<!DOCTYPE html>
<html lang="zh-CN">

<head>
  <meta charset="UTF-8">
  <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=2">
  <meta name="theme-color" content="#222">
  <meta name="generator" content="Hexo 4.2.1">
  <link rel="apple-touch-icon" sizes="180x180" href="/images/apple-touch-icon-next.png">
  <link rel="icon" type="image/png" sizes="32x32" href="/images/favicon-32x32-next.png">
  <link rel="icon" type="image/png" sizes="16x16" href="/images/favicon-16x16-next.png">
  <link rel="mask-icon" href="/images/safari-pinned-tab.svg" color="#222">
  <link rel="stylesheet" href="/css/main.css">
  <link rel="stylesheet" href="/lib/font-awesome/css/all.min.css">
  <link rel="stylesheet" href="/lib/pace/pace-theme-minimal.min.css">
  <script src="/lib/pace/pace.min.js"></script>
  <script id="hexo-configurations">
    var NexT = window.NexT ||
    {};
    var CONFIG = {
      "hostname": "cuiqingcai.com",
      "root": "/",
      "scheme": "Pisces",
      "version": "7.8.0",
      "exturl": false,
      "sidebar":
      {
        "position": "right",
        "width": 360,
        "display": "post",
        "padding": 18,
        "offset": 12,
        "onmobile": false,
        "widgets": [
          {
            "type": "image",
            "name": "阿布云",
            "enable": false,
            "url": "https://www.abuyun.com/http-proxy/introduce.html",
            "src": "https://qiniu.cuiqingcai.com/88au8.jpg",
            "width": "100%"
      },
          {
            "type": "image",
            "name": "天验",
            "enable": true,
            "url": "https://tutorial.lengyue.video/?coupon=12ef4b1a-a3db-11ea-bb37-0242ac130002_cqx_850",
            "src": "https://qiniu.cuiqingcai.com/bco2a.png",
            "width": "100%"
      },
          {
            "type": "image",
            "name": "华为云",
            "enable": false,
            "url": "https://activity.huaweicloud.com/2020_618_promotion/index.html?bpName=5f9f98a29e2c40b780c1793086f29fe2&bindType=1&salesID=wangyubei",
            "src": "https://qiniu.cuiqingcai.com/y42ik.jpg",
            "width": "100%"
      },
          {
            "type": "image",
            "name": "张小鸡",
            "enable": false,
            "url": "http://www.zxiaoji.com/",
            "src": "https://qiniu.cuiqingcai.com/fm72f.png",
            "width": "100%"
      },
          {
            "type": "image",
            "name": "Luminati",
            "src": "https://qiniu.cuiqingcai.com/ikkq9.jpg",
            "url": "https://luminati-china.io/?affiliate=ref_5fbbaaa9647883f5c6f77095",
            "width": "100%",
            "enable": false
      },
          {
            "type": "image",
            "name": "IPIDEA",
            "url": "http://www.ipidea.net/?utm-source=cqc&utm-keyword=?cqc",
            "src": "https://qiniu.cuiqingcai.com/0ywun.png",
            "width": "100%",
            "enable": true
      },
          {
            "type": "tags",
            "name": "标签云",
            "enable": true
      },
          {
            "type": "categories",
            "name": "分类",
            "enable": true
      },
          {
            "type": "friends",
            "name": "友情链接",
            "enable": true
      },
          {
            "type": "hot",
            "name": "猜你喜欢",
            "enable": true
      }]
      },
      "copycode":
      {
        "enable": true,
        "show_result": true,
        "style": "mac"
      },
      "back2top":
      {
        "enable": true,
        "sidebar": false,
        "scrollpercent": true
      },
      "bookmark":
      {
        "enable": false,
        "color": "#222",
        "save": "auto"
      },
      "fancybox": false,
      "mediumzoom": false,
      "lazyload": false,
      "pangu": true,
      "comments":
      {
        "style": "tabs",
        "active": "gitalk",
        "storage": true,
        "lazyload": false,
        "nav": null,
        "activeClass": "gitalk"
      },
      "algolia":
      {
        "hits":
        {
          "per_page": 10
        },
        "labels":
        {
          "input_placeholder": "Search for Posts",
          "hits_empty": "We didn't find any results for the search: ${query}",
          "hits_stats": "${hits} results found in ${time} ms"
        }
      },
      "localsearch":
      {
        "enable": true,
        "trigger": "auto",
        "top_n_per_article": 10,
        "unescape": false,
        "preload": false
      },
      "motion":
      {
        "enable": false,
        "async": false,
        "transition":
        {
          "post_block": "bounceDownIn",
          "post_header": "slideDownIn",
          "post_body": "slideDownIn",
          "coll_header": "slideLeftIn",
          "sidebar": "slideUpIn"
        }
      },
      "path": "search.xml"
    };

  </script>
  <meta name="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
  <meta property="og:type" content="website">
  <meta property="og:title" content="静觅">
  <meta property="og:url" content="https://cuiqingcai.com/page/20/index.html">
  <meta property="og:site_name" content="静觅">
  <meta property="og:description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
  <meta property="og:locale" content="zh_CN">
  <meta property="article:author" content="崔庆才">
  <meta property="article:tag" content="崔庆才">
  <meta property="article:tag" content="静觅">
  <meta property="article:tag" content="PHP">
  <meta property="article:tag" content="Java">
  <meta property="article:tag" content="Python">
  <meta property="article:tag" content="Spider">
  <meta property="article:tag" content="爬虫">
  <meta property="article:tag" content="Web">
  <meta property="article:tag" content="Kubernetes">
  <meta property="article:tag" content="深度学习">
  <meta property="article:tag" content="机器学习">
  <meta property="article:tag" content="数据分析">
  <meta property="article:tag" content="网络">
  <meta property="article:tag" content="IT">
  <meta property="article:tag" content="技术">
  <meta property="article:tag" content="博客">
  <meta name="twitter:card" content="summary">
  <link rel="canonical" href="https://cuiqingcai.com/page/20/">
  <script id="page-configurations">
    // https://hexo.io/docs/variables.html
    CONFIG.page = {
      sidebar: "",
      isHome: true,
      isPost: false,
      lang: 'zh-CN'
    };

  </script>
  <title>静觅丨崔庆才的个人站点</title>
  <meta name="google-site-verification" content="p_bIcnvirkFzG2dYKuNDivKD8-STet5W7D-01woA2fc" />
  <noscript>
    <style>
      .use-motion .brand,
      .use-motion .menu-item,
      .sidebar-inner,
      .use-motion .post-block,
      .use-motion .pagination,
      .use-motion .comments,
      .use-motion .post-header,
      .use-motion .post-body,
      .use-motion .collection-header
      {
        opacity: initial;
      }

      .use-motion .site-title,
      .use-motion .site-subtitle
      {
        opacity: initial;
        top: initial;
      }

      .use-motion .logo-line-before i
      {
        left: initial;
      }

      .use-motion .logo-line-after i
      {
        right: initial;
      }

    </style>
  </noscript>
  <link rel="alternate" href="/atom.xml" title="静觅" type="application/atom+xml">
</head>

<body itemscope itemtype="http://schema.org/WebPage">
  <div class="container">
    <div class="headband"></div>
    <header class="header" itemscope itemtype="http://schema.org/WPHeader">
      <div class="header-inner">
        <div class="site-brand-container">
          <div class="site-nav-toggle">
            <div class="toggle" aria-label="切换导航栏">
              <span class="toggle-line toggle-line-first"></span>
              <span class="toggle-line toggle-line-middle"></span>
              <span class="toggle-line toggle-line-last"></span>
            </div>
          </div>
          <div class="site-meta">
            <a href="/" class="brand" rel="start">
              <span class="logo-line-before"><i></i></span>
              <h1 class="site-title">静觅 <span class="site-subtitle"> 崔庆才的个人站点 </span>
              </h1>
              <span class="logo-line-after"><i></i></span>
            </a>
          </div>
          <div class="site-nav-right">
            <div class="toggle popup-trigger">
              <i class="fa fa-search fa-fw fa-lg"></i>
            </div>
          </div>
        </div>
        <nav class="site-nav">
          <ul id="menu" class="main-menu menu">
            <li class="menu-item menu-item-home">
              <a href="/" rel="section">首页</a>
            </li>
            <li class="menu-item menu-item-archives">
              <a href="/archives/" rel="section">文章列表</a>
            </li>
            <li class="menu-item menu-item-tags">
              <a href="/tags/" rel="section">文章标签</a>
            </li>
            <li class="menu-item menu-item-categories">
              <a href="/categories/" rel="section">文章分类</a>
            </li>
            <li class="menu-item menu-item-about">
              <a href="/about/" rel="section">关于博主</a>
            </li>
            <li class="menu-item menu-item-message">
              <a href="/message/" rel="section">给我留言</a>
            </li>
            <li class="menu-item menu-item-search">
              <a role="button" class="popup-trigger">搜索 </a>
            </li>
          </ul>
        </nav>
        <div class="search-pop-overlay">
          <div class="popup search-popup">
            <div class="search-header">
              <span class="search-icon">
                <i class="fa fa-search"></i>
              </span>
              <div class="search-input-container">
                <input autocomplete="off" autocapitalize="off" placeholder="搜索..." spellcheck="false" type="search" class="search-input">
              </div>
              <span class="popup-btn-close">
                <i class="fa fa-times-circle"></i>
              </span>
            </div>
            <div id="search-result">
              <div id="no-result">
                <i class="fa fa-spinner fa-pulse fa-5x fa-fw"></i>
              </div>
            </div>
          </div>
        </div>
      </div>
    </header>
    <div class="back-to-top">
      <i class="fa fa-arrow-up"></i>
      <span>0%</span>
    </div>
    <div class="reading-progress-bar"></div>
    <main class="main">
      <div class="main-inner">
        <div class="content-wrap">
          <div class="content index posts-expand">
            <div class="carousel">
              <div id="wowslider-container">
                <div class="ws_images">
                  <ul>
                    <li><a target="_blank" href="https://cuiqingcai.com/5052.html"><img title="Python3网络爬虫开发实战教程" src="https://qiniu.cuiqingcai.com/ipy96.jpg" /></a></li>
                    <li><a target="_blank" href="https://t.lagou.com/fRCBRsRCSN6FA"><img title="52讲轻松搞定网络爬虫" src="https://qiniu.cuiqingcai.com/fqq5e.png" /></a></li>
                    <li><a target="_blank" href="https://brightdata.grsm.io/cuiqingcai"><img title="亮网络解锁器" src="https://qiniu.cuiqingcai.com/6qnb7.png" /></a></li>
                    <li><a target="_blank" href="https://cuiqingcai.com/4320.html"><img title="Python3网络爬虫开发视频教程" src="https://qiniu.cuiqingcai.com/bjrny.jpg" /></a></li>
                    <li><a target="_blank" href="https://cuiqingcai.com/5094.html"><img title="爬虫代理哪家强？十大付费代理详细对比评测出炉！" src="https://qiniu.cuiqingcai.com/nifs6.jpg" /></a></li>
                  </ul>
                </div>
                <div class="ws_thumbs">
                  <div>
                    <a target="_blank" href="#"><img src="https://qiniu.cuiqingcai.com/ipy96.jpg" /></a>
                    <a target="_blank" href="#"><img src="https://qiniu.cuiqingcai.com/fqq5e.png" /></a>
                    <a target="_blank" href="#"><img src="https://qiniu.cuiqingcai.com/6qnb7.png" /></a>
                    <a target="_blank" href="#"><img src="https://qiniu.cuiqingcai.com/bjrny.jpg" /></a>
                    <a target="_blank" href="#"><img src="https://qiniu.cuiqingcai.com/nifs6.jpg" /></a>
                  </div>
                </div>
                <div class="ws_shadow"></div>
              </div>
            </div>
            <link rel="stylesheet" href="/lib/wowslide/slide.css">
            <script src="/lib/wowslide/jquery.min.js"></script>
            <script src="/lib/wowslide/slider.js"></script>
            <script>
              jQuery("#wowslider-container").wowSlider(
              {
                effect: "cube",
                prev: "",
                next: "",
                duration: 20 * 100,
                delay: 20 * 100,
                width: 716,
                height: 297,
                autoPlay: true,
                playPause: true,
                stopOnHover: false,
                loop: false,
                bullets: 0,
                caption: true,
                captionEffect: "slide",
                controls: true,
                onBeforeStep: 0,
                images: 0
              });

            </script>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/5189.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/5189.html" class="post-title-link" itemprop="url">[Python3网络爬虫开发实战] 1.3.4-tesserocr的安装</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>在爬虫过程中，难免会遇到各种各样的验证码，而大多数验证码还是图形验证码，这时候我们可以直接用OCR来识别。</p>
                  <h2 id="1-OCR"><a href="#1-OCR" class="headerlink" title="1. OCR"></a>1. OCR</h2>
                  <p>OCR，即Optical Character Recognition，光学字符识别，是指通过扫描字符，然后通过其形状将其翻译成电子文本的过程。对于图形验证码来说，它们都是一些不规则的字符，这些字符确实是由字符稍加扭曲变换得到的内容。</p>
                  <p>例如，对于如图1-22和图1-23所示的验证码，我们可以使用OCR技术来将其转化为电子文本，然后爬虫将识别结果提交给服务器，便可以达到自动识别验证码的过程。</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/1-22.jpg" alt=""></p>
                  <p>图1-22 验证码</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/1-23.jpg" alt="">图1-23 验证码</p>
                  <p>tesserocr是Python的一个OCR识别库，但其实是对tesseract做的一层Python API封装，所以它的核心是tesseract。因此，在安装tesserocr之前，我们需要先安装tesseract。</p>
                  <h2 id="2-相关链接"><a href="#2-相关链接" class="headerlink" title="2. 相关链接"></a>2. 相关链接</h2>
                  <ul>
                    <li>tesserocr GitHub：<a href="https://github.com/sirfz/tesserocr" target="_blank" rel="noopener">https://github.com/sirfz/tesserocr</a></li>
                    <li>tesserocr PyPI：<a href="https://pypi.python.org/pypi/tesserocr" target="_blank" rel="noopener">https://pypi.python.org/pypi/tesserocr</a></li>
                    <li>tesseract下载地址：<a href="http://digi.bib.uni-mannheim.de/tesseract" target="_blank" rel="noopener">http://digi.bib.uni-mannheim.de/tesseract</a></li>
                    <li>tesseract GitHub：<a href="https://github.com/tesseract-ocr/tesseract" target="_blank" rel="noopener">https://github.com/tesseract-ocr/tesseract</a></li>
                    <li>tesseract语言包：<a href="https://github.com/tesseract-ocr/tessdata" target="_blank" rel="noopener">https://github.com/tesseract-ocr/tessdata</a></li>
                    <li>tesseract文档：<a href="https://github.com/tesseract-ocr/tesseract/wiki/Documentation" target="_blank" rel="noopener">https://github.com/tesseract-ocr/tesseract/wiki/Documentation</a></li>
                  </ul>
                  <h2 id="3-Windows下的安装"><a href="#3-Windows下的安装" class="headerlink" title="3. Windows下的安装"></a>3. Windows下的安装</h2>
                  <p>在Windows下，首先需要下载tesseract，它为tesserocr提供了支持。</p>
                  <p>进入下载页面，可以看到有各种.exe文件的下载列表，这里可以选择下载3.0版本。图1-24所示为3.05版本。</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/1-24.jpg" alt="">图1-24 下载页面</p>
                  <p>其中文件名中带有dev的为开发版本，不带dev的为稳定版本，可以选择下载不带dev的版本，例如可以选择下载tesseract-ocr-setup-3.05.01.exe。</p>
                  <p>下载完成后双击，此时会出现如图1-25所示的页面。</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/1-25.jpg" alt="">图1-25 安装页面</p>
                  <p>此时可以勾选Additional language data(download)选项来安装OCR识别支持的语言包，这样OCR便可以识别多国语言。然后一路点击Next按钮即可。</p>
                  <p>接下来，再安装tesserocr即可，此时直接使用pip安装：</p>
                  <figure class="highlight cmake">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">pip3 <span class="keyword">install</span> tesserocr pillow</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h2 id="4-Linux下的安装"><a href="#4-Linux下的安装" class="headerlink" title="4. Linux下的安装"></a>4. Linux下的安装</h2>
                  <p>对于Linux来说，不同系统已经有了不同的发行包了，它可能叫作tesseract-ocr或者tesseract，直接用对应的命令安装即可。</p>
                  <h3 id="Ubuntu、Debian和Deepin"><a href="#Ubuntu、Debian和Deepin" class="headerlink" title="Ubuntu、Debian和Deepin"></a>Ubuntu、Debian和Deepin</h3>
                  <p>在Ubuntu、Debian和Deepin系统下，安装命令如下：</p>
                  <figure class="highlight q">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">sudo apt-<span class="built_in">get</span> install -y tesseract-ocr libtesseract-<span class="built_in">dev</span> libleptonica-<span class="built_in">dev</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h3 id="CentOS、Red-Hat"><a href="#CentOS、Red-Hat" class="headerlink" title="CentOS、Red Hat"></a>CentOS、Red Hat</h3>
                  <p>在CentOS和Red Hat系统下，安装命令如下：</p>
                  <figure class="highlight cmake">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">yum <span class="keyword">install</span> -y tesseract</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>在不同发行版本运行如上命令，即可完成tesseract的安装。</p>
                  <p>安装完成后，便可以调用<code>tesseract</code>命令了。</p>
                  <p>接着，我们查看一下其支持的语言：</p>
                  <figure class="highlight ada">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">tesseract <span class="comment">--list-langs</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果示例：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">List of available languages (<span class="number">3</span>):</span><br><span class="line">eng</span><br><span class="line">osd</span><br><span class="line">equ</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>结果显示它只支持几种语言，如果想要安装多国语言，还需要安装语言包，官方叫作tessdata（其下载链接为：<a href="https://github.com/tesseract-ocr/tessdata" target="_blank" rel="noopener">https://github.com/tesseract-ocr/tessdata</a>）。</p>
                  <p>利用Git命令将其下载下来并迁移到相关目录即可，不同版本的迁移命令如下所示。</p>
                  <p>在Ubuntu、Debian和Deepin系统下的迁移命令如下：</p>
                  <figure class="highlight awk">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">git clone https:<span class="regexp">//gi</span>thub.com<span class="regexp">/tesseract-ocr/</span>tessdata.git</span><br><span class="line">sudo mv tessdata<span class="regexp">/* /u</span>sr<span class="regexp">/share/</span>tesseract-ocr<span class="regexp">/tessdata</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>在CentOS和Red Hat系统下的迁移命令如下：</p>
                  <figure class="highlight awk">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">git clone https:<span class="regexp">//gi</span>thub.com<span class="regexp">/tesseract-ocr/</span>tessdata.git</span><br><span class="line">sudo mv tessdata<span class="regexp">/* /u</span>sr<span class="regexp">/share/</span>tesseract<span class="regexp">/tessdata</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样就可以将下载下来的语言包全部安装了。</p>
                  <p>这时我们重新运行列出所有语言的命令：</p>
                  <figure class="highlight ada">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">tesseract <span class="comment">--list-langs</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>结果如下：</p>
                  <figure class="highlight armasm">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="symbol">List</span> of available languages (<span class="number">107</span>):</span><br><span class="line"><span class="symbol">afr</span></span><br><span class="line"><span class="symbol">amh</span></span><br><span class="line"><span class="symbol">ara</span></span><br><span class="line"><span class="symbol">asm</span></span><br><span class="line"><span class="symbol">aze</span></span><br><span class="line"><span class="symbol">aze_cyrl</span></span><br><span class="line"><span class="keyword">bel</span></span><br><span class="line"><span class="keyword">ben</span></span><br><span class="line"><span class="keyword">bod</span></span><br><span class="line"><span class="keyword">bos</span></span><br><span class="line"><span class="keyword">bul</span></span><br><span class="line"><span class="keyword">cat</span></span><br><span class="line"><span class="keyword">ceb</span></span><br><span class="line"><span class="keyword">ces</span></span><br><span class="line"><span class="keyword">chi_sim</span></span><br><span class="line"><span class="keyword">chi_tra</span></span><br><span class="line"><span class="keyword">...</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以发现，这里列出的语言就多了很多，比如chi_sim就代表简体中文，这就证明语言包安装成功了。</p>
                  <p>接下来再安装tesserocr即可，这里直接使用pip安装：</p>
                  <figure class="highlight cmake">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">pip3 <span class="keyword">install</span> tesserocr pillow</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h2 id="5-Mac下的安装"><a href="#5-Mac下的安装" class="headerlink" title="5. Mac下的安装"></a>5. Mac下的安装</h2>
                  <p>在Mac下，我们首先使用Homebrew安装ImageMagick和tesseract库：</p>
                  <figure class="highlight mipsasm">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">brew </span><span class="keyword">install </span>imagemagick </span><br><span class="line"><span class="keyword">brew </span><span class="keyword">install </span>tesseract --all-languages</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>接下来再安装tesserocr即可：</p>
                  <figure class="highlight cmake">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">pip3 <span class="keyword">install</span> tesserocr pillow</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样我们便完成了tesserocr的安装。</p>
                  <h2 id="6-验证安装"><a href="#6-验证安装" class="headerlink" title="6. 验证安装"></a>6. 验证安装</h2>
                  <p>接下来，我们可以使用tesseract和tesserocr来分别进行测试。</p>
                  <p>下面我们以如图1-26所示的图片为样例进行测试。</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/1-26.jpg" alt="">图1-26 测试样例</p>
                  <p>该图片的链接为<a href="https://raw.githubusercontent.com/Python3WebSpider/TestTess/master/image.png" target="_blank" rel="noopener">https://raw.githubusercontent.com/Python3WebSpider/TestTess/master/image.png</a>，可以直接保存或下载。</p>
                  <p>首先用命令行进行测试，将图片下载下来并保存为image.png，然后用<code>tesseract</code>命令测试：</p>
                  <figure class="highlight applescript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">tesseract image.png <span class="literal">result</span> -l eng &amp;&amp; cat <span class="literal">result</span>.txt</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">Tesseract Open Source OCR Engine v3<span class="number">.05</span><span class="number">.01</span> with Leptonica</span><br><span class="line">Python3WebSpider</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们调用了<code>tesseract</code>命令，其中第一个参数为图片名称，第二个参数<code>result</code>为结果保存的目标文件名称，<code>\-l</code>指定使用的语言包，在此使用英文（<code>eng</code>）。然后，再用<code>cat</code>命令将结果输出。</p>
                  <p>运行结果便是图片的识别结果：<code>Python3WebSpider</code>。可以看到，这时已经成功将图片文字转为电子文本了。</p>
                  <p>然后还可以利用Python代码来测试，这里就需要借助于tesserocr库了，测试代码如下：</p>
                  <figure class="highlight processing">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">import</span> tesserocr</span><br><span class="line">from PIL <span class="keyword">import</span> Image</span><br><span class="line"><span class="built_in">image</span> = Image.<span class="built_in">open</span>(<span class="string">'image.png'</span>)</span><br><span class="line"><span class="built_in">print</span>(tesserocr.image_to_text(<span class="built_in">image</span>))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>我们首先利用<code>Image</code>读取了图片文件，然后调用了<code>tesserocr</code>的<code>image_to_text()</code>方法，再将其识别结果输出。</p>
                  <p>运行结果如下：</p>
                  <figure class="highlight gcode">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">Pytho<span class="symbol">n3</span>WebSpider</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>另外，我们还可以直接调用<code>file_to_text()</code>方法，这可以达到同样的效果：</p>
                  <figure class="highlight lisp">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import tesserocr</span><br><span class="line">print(<span class="name">tesserocr</span>.file_to_text('image.png'))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果：</p>
                  <figure class="highlight gcode">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">Pytho<span class="symbol">n3</span>WebSpider</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>如果成功输出结果，则证明tesseract和tesserocr都已经安装成功。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-01-25 14:56:13" itemprop="dateCreated datePublished" datetime="2018-01-25T14:56:13+08:00">2018-01-25</time>
                </span>
                <span id="/5189.html" class="post-meta-item leancloud_visitors" data-flag-title="[Python3网络爬虫开发实战] 1.3.4-tesserocr的安装" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>3.3k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>3 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/5186.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/5186.html" class="post-title-link" itemprop="url">[Python3网络爬虫开发实战] 1.3.3-pyquery的安装</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>pyquery同样是一个强大的网页解析工具，它提供了和jQuery类似的语法来解析HTML文档，支持CSS选择器，使用非常方便。本节中，我们就来了解一下它的安装方式。</p>
                  <h2 id="1-相关链接"><a href="#1-相关链接" class="headerlink" title="1. 相关链接"></a>1. 相关链接</h2>
                  <ul>
                    <li>GitHub：<a href="https://github.com/gawel/pyquery" target="_blank" rel="noopener">https://github.com/gawel/pyquery</a></li>
                    <li>PyPI：<a href="https://pypi.python.org/pypi/pyquery" target="_blank" rel="noopener">https://pypi.python.org/pypi/pyquery</a></li>
                    <li>官方文档：<a href="http://pyquery.readthedocs.io/" target="_blank" rel="noopener">http://pyquery.readthedocs.io</a></li>
                  </ul>
                  <h2 id="2-pip安装"><a href="#2-pip安装" class="headerlink" title="2. pip安装"></a>2. pip安装</h2>
                  <p>这里推荐使用pip安装，命令如下：</p>
                  <figure class="highlight cmake">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">pip3 <span class="keyword">install</span> pyquery</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>命令执行完毕之后即可完成安装。</p>
                  <h2 id="3-wheel安装"><a href="#3-wheel安装" class="headerlink" title="3. wheel安装"></a>3. wheel安装</h2>
                  <p>当然，我们也可以到PyPI（<a href="https://pypi.python.org/pypi/pyquery/#downloads" target="_blank" rel="noopener">https://pypi.python.org/pypi/pyquery/#downloads</a>）下载对应的wheel文件安装。比如如果当前版本为1.2.17，则下载的文件名称为pyquery-1.2.17-py2.py3-none-any.whl，此时下载到本地再进行pip安装即可，命令如下：</p>
                  <figure class="highlight css">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="selector-tag">pip3</span> <span class="selector-tag">install</span> <span class="selector-tag">pyquery-1</span><span class="selector-class">.2</span><span class="selector-class">.17-py2</span><span class="selector-class">.py3-none-any</span><span class="selector-class">.whl</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h2 id="4-验证安装"><a href="#4-验证安装" class="headerlink" title="4. 验证安装"></a>4. 验证安装</h2>
                  <p>安装完成之后，可以在Python命令行下测试：</p>
                  <figure class="highlight ruby">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">$ python3</span><br><span class="line"><span class="meta">&gt;&gt;</span>&gt; import pyquery</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>如果没有错误报出，则证明库已经安装好了。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-01-25 14:53:07" itemprop="dateCreated datePublished" datetime="2018-01-25T14:53:07+08:00">2018-01-25</time>
                </span>
                <span id="/5186.html" class="post-meta-item leancloud_visitors" data-flag-title="[Python3网络爬虫开发实战] 1.3.3-pyquery的安装" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>562</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>1 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/5183.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/5183.html" class="post-title-link" itemprop="url">[Python3网络爬虫开发实战] 1.3.2-Beautiful Soup的安装</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>Beautiful Soup是Python的一个HTML或XML的解析库，我们可以用它来方便地从网页中提取数据。它拥有强大的API和多样的解析方式，本节就来了解下它的安装方式。</p>
                  <h2 id="1-相关链接"><a href="#1-相关链接" class="headerlink" title="1. 相关链接"></a>1. 相关链接</h2>
                  <ul>
                    <li>官方文档：<a href="https://www.crummy.com/software/BeautifulSoup/bs4/doc" target="_blank" rel="noopener">https://www.crummy.com/software/BeautifulSoup/bs4/doc</a></li>
                    <li>中文文档：<a href="https://www.crummy.com/software/BeautifulSoup/bs4/doc.zh" target="_blank" rel="noopener">https://www.crummy.com/software/BeautifulSoup/bs4/doc.zh</a></li>
                    <li>PyPI：<a href="https://pypi.python.org/pypi/beautifulsoup4" target="_blank" rel="noopener">https://pypi.python.org/pypi/beautifulsoup4</a></li>
                  </ul>
                  <h2 id="2-准备工作"><a href="#2-准备工作" class="headerlink" title="2. 准备工作"></a>2. 准备工作</h2>
                  <p>Beautiful Soup的HTML和XML解析器是依赖于lxml库的，所以在此之前请确保已经成功安装好了lxml库，具体的安装方式参见上节。</p>
                  <h2 id="3-pip安装"><a href="#3-pip安装" class="headerlink" title="3. pip安装"></a>3. pip安装</h2>
                  <p>目前，Beautiful Soup的最新版本是4.x版本，之前的版本已经停止开发了。这里推荐使用pip来安装，安装命令如下：</p>
                  <figure class="highlight smali">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">pip3 install beautifulsoup4</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>命令执行完毕之后即可完成安装。</p>
                  <h2 id="4-wheel安装"><a href="#4-wheel安装" class="headerlink" title="4. wheel安装"></a>4. wheel安装</h2>
                  <p>当然，我们也可以从PyPI下载wheel文件安装，链接如下：<a href="https://pypi.python.org/pypi/beautifulsoup4" target="_blank" rel="noopener">https://pypi.python.org/pypi/beautifulsoup4</a></p>
                  <p>然后使用pip安装wheel文件即可。</p>
                  <h2 id="5-验证安装"><a href="#5-验证安装" class="headerlink" title="5. 验证安装"></a>5. 验证安装</h2>
                  <p>安装完成之后，可以运行下面的代码验证一下：</p>
                  <figure class="highlight moonscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> bs4 <span class="keyword">import</span> BeautifulSoup</span><br><span class="line">soup = BeautifulSoup(<span class="string">'&lt;p&gt;Hello&lt;/p&gt;'</span>, <span class="string">'lxml'</span>)</span><br><span class="line"><span class="built_in">print</span>(soup.p.<span class="built_in">string</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">Hello</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>如果运行结果一致，则证明安装成功。</p>
                  <p>注意，这里我们虽然安装的是beautifulsoup4这个包，但是在引入的时候却是bs4。这是因为这个包源代码本身的库文件夹名称就是bs4，所以安装完成之后，这个库文件夹就被移入到本机Python3的lib库里，所以识别到的库文件名就叫作bs4。</p>
                  <p>因此，包本身的名称和我们使用时导入的包的名称并不一定是一致的。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-01-25 14:45:43" itemprop="dateCreated datePublished" datetime="2018-01-25T14:45:43+08:00">2018-01-25</time>
                </span>
                <span id="/5183.html" class="post-meta-item leancloud_visitors" data-flag-title="[Python3网络爬虫开发实战] 1.3.2-Beautiful Soup的安装" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>860</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>1 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/5180.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/5180.html" class="post-title-link" itemprop="url">[Python3网络爬虫开发实战] 1.3.1-lxml的安装</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>lxml是Python的一个解析库，支持HTML和XML的解析，支持XPath解析方式，而且解析效率非常高。本节中，我们了解一下lxml的安装方式，这主要从Windows、Linux和Mac三大平台来介绍。</p>
                  <h2 id="1-相关链接"><a href="#1-相关链接" class="headerlink" title="1. 相关链接"></a>1. 相关链接</h2>
                  <ul>
                    <li>官方网站：<a href="http://lxml.de/" target="_blank" rel="noopener">http://lxml.de</a></li>
                    <li>GitHub：<a href="https://github.com/lxml/lxml" target="_blank" rel="noopener">https://github.com/lxml/lxml</a></li>
                    <li>PyPI：<a href="https://pypi.python.org/pypi/lxml" target="_blank" rel="noopener">https://pypi.python.org/pypi/lxml</a></li>
                  </ul>
                  <h2 id="2-Windows下的安装"><a href="#2-Windows下的安装" class="headerlink" title="2. Windows下的安装"></a>2. Windows下的安装</h2>
                  <p>在Windows下，可以先尝试利用pip安装，此时直接执行如下命令即可：</p>
                  <figure class="highlight cmake">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">pip3 <span class="keyword">install</span> lxml</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>如果没有任何报错，则证明安装成功。</p>
                  <p>如果出现报错，比如提示缺少libxml2库等信息，可以采用wheel方式安装。</p>
                  <p>推荐直接到这里（链接为：<a href="http://www.lfd.uci.edu/~gohlke/pythonlibs/#lxml" target="_blank" rel="noopener">http://www.lfd.uci.edu/~gohlke/pythonlibs/#lxml</a>）下载对应的wheel文件，找到本地安装Python版本和系统对应的lxml版本，例如Windows 64位、Python 3.6，就选择lxml‑3.8.0‑cp36‑cp36m‑win_amd64.whl，将其下载到本地。</p>
                  <p>然后利用pip安装即可，命令如下：</p>
                  <figure class="highlight css">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="selector-tag">pip3</span> <span class="selector-tag">install</span> <span class="selector-tag">lxml</span>‑3<span class="selector-class">.8</span><span class="selector-class">.0</span>‑<span class="selector-tag">cp36</span>‑<span class="selector-tag">cp36m</span>‑<span class="selector-tag">win_amd64</span><span class="selector-class">.whl</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样我们就可以成功安装lxml了。</p>
                  <h2 id="3-Linux下的安装"><a href="#3-Linux下的安装" class="headerlink" title="3. Linux下的安装"></a>3. Linux下的安装</h2>
                  <p>在Linux平台下安装问题不大，同样可以先尝试pip安装，命令如下：</p>
                  <figure class="highlight cmake">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">pip3 <span class="keyword">install</span> lxml</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>如果报错，可以尝试下面的解决方案。</p>
                  <h3 id="CentOS、Red-Hat"><a href="#CentOS、Red-Hat" class="headerlink" title="CentOS、Red Hat"></a>CentOS、Red Hat</h3>
                  <p>对于此类系统，报错主要是因为缺少必要的库。</p>
                  <p>执行如下命令安装所需的库即可：</p>
                  <figure class="highlight gml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">sudo yum groupinstall -<span class="symbol">y</span> development tools</span><br><span class="line">sudo yum install -<span class="symbol">y</span> epel-release libxslt-devel libxml2-devel openssl-devel</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>主要是libxslt-devel和libxml2-devel这两个库，lxml依赖它们。安装好之后，重新尝试pip安装即可。</p>
                  <h3 id="Ubuntu、Debian和Deepin"><a href="#Ubuntu、Debian和Deepin" class="headerlink" title="Ubuntu、Debian和Deepin"></a>Ubuntu、Debian和Deepin</h3>
                  <p>在这些系统下，报错的原因同样可能是缺少了必要的类库，执行如下命令安装：</p>
                  <figure class="highlight q">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">sudo apt-<span class="built_in">get</span> install -y python3-<span class="built_in">dev</span> build-essential libssl-<span class="built_in">dev</span> libffi-<span class="built_in">dev</span> libxml2 libxml2-<span class="built_in">dev</span> libxslt1-<span class="built_in">dev</span> zlib1g-<span class="built_in">dev</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>安装好之后，重新尝试pip安装即可。</p>
                  <h2 id="4-Mac下的安装"><a href="#4-Mac下的安装" class="headerlink" title="4. Mac下的安装"></a>4. Mac下的安装</h2>
                  <p>在Mac平台下，仍然可以首先尝试pip安装，命令如下：</p>
                  <figure class="highlight cmake">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">pip3 <span class="keyword">install</span> lxml</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>如果产生错误，可以执行如下命令将必要的类库安装：</p>
                  <figure class="highlight ada">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">xcode-<span class="keyword">select</span> <span class="comment">--install</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>之后再重新尝试pip安装，就没有问题了。</p>
                  <p>lxml是一个非常重要的库，后面的Beautiful Soup、Scrapy框架都需要用到此库，所以请一定安装成功。</p>
                  <h2 id="5-验证安装"><a href="#5-验证安装" class="headerlink" title="5. 验证安装"></a>5. 验证安装</h2>
                  <p>安装完成之后，可以在Python命令行下测试：</p>
                  <figure class="highlight ruby">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">$ python3</span><br><span class="line"><span class="meta">&gt;&gt;</span>&gt; import lxml</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>如果没有错误报出，则证明库已经安装好了。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-01-25 14:41:12" itemprop="dateCreated datePublished" datetime="2018-01-25T14:41:12+08:00">2018-01-25</time>
                </span>
                <span id="/5180.html" class="post-meta-item leancloud_visitors" data-flag-title="[Python3网络爬虫开发实战] 1.3.1-lxml的安装" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>1.3k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>1 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/5168.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/5168.html" class="post-title-link" itemprop="url">[Python3网络爬虫开发实战] 1.3-解析库的安装</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>抓取网页代码之后，下一步就是从网页中提取信息。提取信息的方式有多种多样，可以使用正则来提取，但是写起来相对比较烦琐。这里还有许多强大的解析库，如lxml、Beautiful Soup、pyquery等。此外，还提供了非常强大的解析方法，如XPath解析和CSS选择器解析等，利用它们，我们可以高效便捷地从网页中提取有效信息。</p>
                  <p>本节中，我们就来介绍一下这些库的安装过程。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-01-25 14:35:54" itemprop="dateCreated datePublished" datetime="2018-01-25T14:35:54+08:00">2018-01-25</time>
                </span>
                <span id="/5168.html" class="post-meta-item leancloud_visitors" data-flag-title="[Python3网络爬虫开发实战] 1.3-解析库的安装" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>182</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>1 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/5163.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/5163.html" class="post-title-link" itemprop="url">[Python3网络爬虫开发实战] 1.2.6-aiohttp的安装</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>之前介绍的Requests库是一个阻塞式HTTP请求库，当我们发出一个请求后，程序会一直等待服务器响应，直到得到响应后，程序才会进行下一步处理。其实，这个过程比较耗费资源。如果程序可以在这个等待过程中做一些其他的事情，如进行请求的调度、响应的处理等，那么爬取效率一定会大大提高。</p>
                  <p>aiohttp就是这样一个提供异步Web服务的库，从Python 3.5版本开始，Python中加入了<code>async</code>/<code>await</code>关键字，使得回调的写法更加直观和人性化。aiohttp的异步操作借助于<code>async</code>/<code>await</code>关键字的写法变得更加简洁，架构更加清晰。使用异步请求库进行数据抓取时，会大大提高效率，下面我们来看一下这个库的安装方法。</p>
                  <h2 id="1-相关链接"><a href="#1-相关链接" class="headerlink" title="1. 相关链接"></a>1. 相关链接</h2>
                  <ul>
                    <li>官方文档：<a href="http://aiohttp.readthedocs.io/en/stable" target="_blank" rel="noopener">http://aiohttp.readthedocs.io/en/stable</a></li>
                    <li>GitHub：<a href="https://github.com/aio-libs/aiohttp" target="_blank" rel="noopener">https://github.com/aio-libs/aiohttp</a></li>
                    <li>PyPI：<a href="https://pypi.python.org/pypi/aiohttp" target="_blank" rel="noopener">https://pypi.python.org/pypi/aiohttp</a></li>
                  </ul>
                  <h2 id="2-pip安装"><a href="#2-pip安装" class="headerlink" title="2. pip安装"></a>2. pip安装</h2>
                  <p>这里推荐使用pip安装，命令如下：</p>
                  <figure class="highlight cmake">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">pip3 <span class="keyword">install</span> aiohttp</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>另外，官方还推荐安装如下两个库：一个是字符编码检测库cchardet，另一个是加速DNS的解析库aiodns。安装命令如下：</p>
                  <figure class="highlight cmake">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">pip3 <span class="keyword">install</span> cchardet aiodns</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h2 id="3-测试安装"><a href="#3-测试安装" class="headerlink" title="3. 测试安装"></a>3. 测试安装</h2>
                  <p>安装完成之后，可以在Python命令行下测试：</p>
                  <figure class="highlight ruby">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">$ python3</span><br><span class="line"><span class="meta">&gt;&gt;</span>&gt; import aiohttp</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>如果没有错误报出，则证明库已经安装好了。</p>
                  <h2 id="4-结语"><a href="#4-结语" class="headerlink" title="4. 结语"></a>4. 结语</h2>
                  <p>我们会在后面的实例中用到这个库，比如维护一个代理池时，利用异步方式检测大量代理的运行状况，会极大地提升效率。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-01-25 13:42:51" itemprop="dateCreated datePublished" datetime="2018-01-25T13:42:51+08:00">2018-01-25</time>
                </span>
                <span id="/5163.html" class="post-meta-item leancloud_visitors" data-flag-title="[Python3网络爬虫开发实战] 1.2.6-aiohttp的安装" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>712</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>1 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/5159.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/5159.html" class="post-title-link" itemprop="url">[Python3网络爬虫开发实战] 1.2.5-PhantomJS的安装</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>PhantomJS是一个无界面的、可脚本编程的WebKit浏览器引擎，它原生支持多种Web标准：DOM操作、CSS选择器、JSON、Canvas以及SVG。</p>
                  <p>Selenium支持PhantomJS，这样在运行的时候就不会再弹出一个浏览器了。而且PhantomJS的运行效率也很高，还支持各种参数配置，使用非常方便。下面我们就来了解一下PhantomJS的安装过程。</p>
                  <h2 id="1-相关链接"><a href="#1-相关链接" class="headerlink" title="1. 相关链接"></a>1. 相关链接</h2>
                  <ul>
                    <li>官方网站：<a href="http://phantomjs.org/" target="_blank" rel="noopener">http://phantomjs.org</a></li>
                    <li>官方文档：<a href="http://phantomjs.org/quick-start.html" target="_blank" rel="noopener">http://phantomjs.org/quick-start.html</a></li>
                    <li>下载地址：<a href="http://phantomjs.org/download.html" target="_blank" rel="noopener">http://phantomjs.org/download.html</a></li>
                    <li>API接口说明：<a href="http://phantomjs.org/api/command-line.html" target="_blank" rel="noopener">http://phantomjs.org/api/command-line.html</a></li>
                  </ul>
                  <h2 id="2-下载PhantomJS"><a href="#2-下载PhantomJS" class="headerlink" title="2. 下载PhantomJS"></a>2. 下载PhantomJS</h2>
                  <p>我们需要在官方网站下载对应的安装包，PhantomJS支持多种操作系统，比如Windows、Linux、Mac、FreeBSD等，我们可以选择对应的平台并将安装包下载下来。</p>
                  <p>下载完成后，将PhantomJS可执行文件所在的路径配置到环境变量里。比如在Windows下，将下载的文件解压之后并打开，会看到一个bin文件夹，里面包括一个可执行文件phantomjs.exe，我们需要将它直接放在配置好环境变量的路径下或者将它所在的路径配置到环境变量里。比如，我们既可以将它直接复制到Python的Scripts文件夹，也可以将它所在的bin目录加入到环境变量。</p>
                  <p>Windows下环境变量的配置可以参见1.1节，Linux及Mac环境变量的配置可以参见1.2.3节，在此不再赘述，关键在于将PhantomJS的可执行文件所在路径配置到环境变量里。</p>
                  <p>配置成功后，可以在命令行下测试一下，输入：</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">phantomjs</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>如果可以进入到PhantomJS的命令行，那就证明配置完成了，如图1-21所示。<img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/1-21.jpg" alt=""></p>
                  <p>图1-21 控制台</p>
                  <h2 id="3-验证安装"><a href="#3-验证安装" class="headerlink" title="3. 验证安装"></a>3. 验证安装</h2>
                  <p>在Selenium中使用的话，我们只需要将Chrome切换为PhantomJS即可：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> selenium import webdriver</span><br><span class="line">browser = webdriver.PhantomJS()</span><br><span class="line">browser.<span class="builtin-name">get</span>(<span class="string">'https://www.baidu.com'</span>)</span><br><span class="line"><span class="builtin-name">print</span>(browser.current_url)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行之后，我们就不会发现有浏览器弹出了，但实际上PhantomJS已经运行起来了。这里我们访问了百度，然后将当前的URL打印出来。</p>
                  <p>控制台的输出如下：</p>
                  <figure class="highlight dts">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="symbol">https:</span><span class="comment">//www.baidu.com/</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>如此一来，我们便完成了PhantomJS的配置，后面可以利用它来完成一些页面的抓取。</p>
                  <p>这里我们介绍了Selenium对应的三大主流浏览器的对接方式，后面我们会对Selenium及各个浏览器的对接方法进行更加深入的探究。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-01-25 13:38:03" itemprop="dateCreated datePublished" datetime="2018-01-25T13:38:03+08:00">2018-01-25</time>
                </span>
                <span id="/5159.html" class="post-meta-item leancloud_visitors" data-flag-title="[Python3网络爬虫开发实战] 1.2.5-PhantomJS的安装" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>1.2k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>1 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/5153.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/5153.html" class="post-title-link" itemprop="url">[Python3网络爬虫开发实战] 1.2.4-GeckoDriver的安装</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>上一节中，我们了解了ChromeDriver的配置方法，配置完成之后便可以用Selenium驱动Chrome浏览器来做相应网页的抓取。</p>
                  <p>那么对于Firefox来说，也可以使用同样的方式完成Selenium的对接，这时需要安装另一个驱动GeckoDriver。</p>
                  <p>本节中，我们来介绍一下GeckoDriver的安装过程。</p>
                  <h2 id="1-相关链接"><a href="#1-相关链接" class="headerlink" title="1. 相关链接"></a>1. 相关链接</h2>
                  <ul>
                    <li>GitHub：<a href="https://github.com/mozilla/geckodriver" target="_blank" rel="noopener">https://github.com/mozilla/geckodriver</a></li>
                    <li>下载地址：<a href="https://github.com/mozilla/geckodriver/releases" target="_blank" rel="noopener">https://github.com/mozilla/geckodriver/releases</a></li>
                  </ul>
                  <h2 id="2-准备工作"><a href="#2-准备工作" class="headerlink" title="2. 准备工作"></a>2. 准备工作</h2>
                  <p>在这之前请确保已经正确安装好了Firefox浏览器并可以正常运行，安装过程不再赘述。</p>
                  <h2 id="3-下载GeckoDriver"><a href="#3-下载GeckoDriver" class="headerlink" title="3. 下载GeckoDriver"></a>3. 下载GeckoDriver</h2>
                  <p>我们可以在GitHub上找到GeckoDriver的发行版本，当前最新版本为0.18，下载页面如图1-18所示。<img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/1-18.png" alt="">图1-18 GeckoDriver下载页面</p>
                  <p>这里可以在不同的平台上下载，如Windows、Mac、Linux、ARM等平台，我们可以根据自己的系统和位数选择对应的驱动下载，若是Windows 64位，就下载geckodriver-v0.18.0-win64.zip。</p>
                  <h2 id="4-环境变量配置"><a href="#4-环境变量配置" class="headerlink" title="4. 环境变量配置"></a>4. 环境变量配置</h2>
                  <p>在Windows下，可以直接将geckodriver.exe文件拖到Python的Scripts目录下，如图1-19所示。<img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/1-19.jpg" alt=""></p>
                  <p>图1-19 将geckodriver.exe文件拖到Python Scripts目录</p>
                  <p>此外，也可以单独将其所在路径配置到环境变量，具体的配置方法请参1.1节。</p>
                  <p>在Linux和Mac下，需要将可执行文件配置到环境变量或将文件移动到属于环境变量的目录里。</p>
                  <p>例如，要移动文件到/usr/bin目录。首先在命令行模式下进入其所在路径，然后将其移动到/usr/bin：</p>
                  <figure class="highlight awk">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">sudo mv geckodriver <span class="regexp">/usr/</span>bin</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>当然，也可以将GeckoDriver配置到$PATH。首先，可以将可执行文件放到某一目录，目录可以任意选择，例如将当前可执行文件放在/usr/local/geckodriver目录下。接下来可以修改~/.profile文件，命令如下：</p>
                  <figure class="highlight vim">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">vi</span> ~/.<span class="keyword">profile</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>然后添加如下一句配置：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="builtin-name">export</span> <span class="attribute">PATH</span>=<span class="string">"<span class="variable">$PATH</span>:/usr/local/geckodriver"</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>保存后执行如下命令即可完成配置：</p>
                  <figure class="highlight vim">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">source</span> ~/.<span class="keyword">profile</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h2 id="5-验证安装"><a href="#5-验证安装" class="headerlink" title="5. 验证安装"></a>5. 验证安装</h2>
                  <p>配置完成后，就可以在命令行下直接执行<code>geckodriver</code>命令测试：</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">geckodriver</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这时如果控制台有类似图1-20所示的输出，则证明GeckoDriver的环境变量配置好了。<img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/1-20.jpg" alt=""></p>
                  <p>图1-20 控制台输出</p>
                  <p>随后执行如下Python代码，在程序中测试一下：</p>
                  <figure class="highlight armasm">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="symbol">from</span> <span class="keyword">selenium </span><span class="meta">import</span> webdriver</span><br><span class="line"><span class="keyword">browser </span>= webdriver.Firefox()</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行之后，若弹出一个空白的Firefox浏览器，则证明所有的配置都没有问题；如果没有弹出，请检查之前的每一步配置。</p>
                  <p>如果没有问题，接下来就可以利用Firefox配合Selenium来做网页抓取了。</p>
                  <h2 id="6-结语"><a href="#6-结语" class="headerlink" title="6. 结语"></a>6. 结语</h2>
                  <p>现在我们就可以使用Chrome或Firefox进行网页抓取了，但是这样可能有个不方便之处：因为程序运行过程中需要一直开着浏览器，在爬取网页的过程中浏览器可能一直动来动去。目前最新的Chrome浏览器版本已经支持无界面模式了，但如果版本较旧的话，就不支持。所以这里还有另一种选择，那就是安装一个无界面浏览器PhantomJS，此时抓取过程会在后台运行，不会再有窗口出现。在下一节中，我们就来了解一下PhantomJS的相关安装方法。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-01-25 13:29:52" itemprop="dateCreated datePublished" datetime="2018-01-25T13:29:52+08:00">2018-01-25</time>
                </span>
                <span id="/5153.html" class="post-meta-item leancloud_visitors" data-flag-title="[Python3网络爬虫开发实战] 1.2.4-GeckoDriver的安装" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>1.5k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>1 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/5135.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/5135.html" class="post-title-link" itemprop="url">[Python3网络爬虫开发实战] 1.2.3-ChromeDriver的安装</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>前面我们成功安装好了Selenium库，但是它是一个自动化测试工具，需要浏览器来配合使用，本节中我们就介绍一下Chrome浏览器及ChromeDriver驱动的配置。</p>
                  <p>首先，下载Chrome浏览器，方法有很多，在此不再赘述。</p>
                  <p>随后安装ChromeDriver。因为只有安装ChromeDriver，才能驱动Chrome浏览器完成相应的操作。下面我们来介绍下怎样安装ChromeDriver。</p>
                  <h2 id="1-相关链接"><a href="#1-相关链接" class="headerlink" title="1. 相关链接"></a>1. 相关链接</h2>
                  <ul>
                    <li>官方网站：<a href="https://sites.google.com/a/chromium.org/chromedriver" target="_blank" rel="noopener">https://sites.google.com/a/chromium.org/chromedriver</a></li>
                    <li>下载地址：<a href="https://chromedriver.storage.googleapis.com/index.html" target="_blank" rel="noopener">https://chromedriver.storage.googleapis.com/index.html</a></li>
                  </ul>
                  <h2 id="2-准备工作"><a href="#2-准备工作" class="headerlink" title="2. 准备工作"></a>2. 准备工作</h2>
                  <p>在这之前请确保已经正确安装好了Chrome浏览器并可以正常运行，安装过程不再赘述。</p>
                  <h2 id="3-查看版本"><a href="#3-查看版本" class="headerlink" title="3. 查看版本"></a>3. 查看版本</h2>
                  <p>点击Chrome菜单“帮助”→“关于Google Chrome”，即可查看Chrome的版本号，如图1-14所示。<img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/1-14.jpg" alt=""></p>
                  <p>图1-14 Chrome版本号</p>
                  <p>这里我的Chrome版本是58.0。</p>
                  <p>请记住Chrome版本号，因为选择ChromeDriver版本时需要用到。</p>
                  <h2 id="4-下载ChromeDriver"><a href="#4-下载ChromeDriver" class="headerlink" title="4. 下载ChromeDriver"></a>4. 下载ChromeDriver</h2>
                  <p>打开ChromeDriver的官方网站，可以看到最新版本为2.31，其支持的Chrome浏览器版本为58~60，官网页面如图1-15所示。</p>
                  <blockquote>
                    <p>更新：现在 2020 年，Chrome 版本已经更新到 80+，请以最新的 ChromeDriver 为准！<a href="https://chromedriver.chromium.org/downloads" target="_blank" rel="noopener">https://chromedriver.chromium.org/downloads</a></p>
                  </blockquote>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/1-15.jpg" alt=""></p>
                  <p>图1-15 官网页面</p>
                  <p>如果你的Chrome版本号是58~60，那么可以选择此版本下载。</p>
                  <p>如果你的Chrome版本号不在此范围，可以继续查看之前的ChromeDriver版本。每个版本都有相应的支持Chrome版本的介绍，请找好自己的Chrome浏览器版本对应的ChromeDriver版本再下载，否则可能无法正常工作。</p>
                  <p>找好对应的版本号后，随后到ChromeDriver镜像站下载对应的安装包即可：<a href="https://chromedriver.storage.googleapis.com/index.html" target="_blank" rel="noopener">https://chromedriver.storage.googleapis.com/index.html</a>。在不同平台下，可以下载不同的安装包。</p>
                  <h2 id="5-环境变量配置"><a href="#5-环境变量配置" class="headerlink" title="5. 环境变量配置"></a>5. 环境变量配置</h2>
                  <p>下载完成后，将ChromeDriver的可执行文件配置到环境变量下。</p>
                  <p>在Windows下，建议直接将chromedriver.exe文件拖到Python的Scripts目录下，如图1-16所示。</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/1-16.jpg" alt=""></p>
                  <p>图1-16 Python Scripts目录</p>
                  <p>此外，也可以单独将其所在路径配置到环境变量，具体的配置方法请参见1.1节。</p>
                  <p>在Linux和Mac下，需要将可执行文件配置到环境变量或将文件移动到属于环境变量的目录里。</p>
                  <p>例如，要移动文件到/usr/bin目录。首先，需要在命令行模式下进入其所在路径，然后将其移动到/usr/bin：</p>
                  <figure class="highlight awk">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">sudo mv chromedriver <span class="regexp">/usr/</span>bin</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>当然，也可以将ChromeDriver配置到$PATH。首先，可以将可执行文件放到某一目录，目录可以任意选择，例如将当前可执行文件放在/usr/local/chromedriver目录下，接下来可以修改~/.profile文件，相关命令如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="builtin-name">export</span> <span class="attribute">PATH</span>=<span class="string">"<span class="variable">$PATH</span>:/usr/local/chromedriver"</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>保存后执行如下命令：</p>
                  <figure class="highlight vim">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">source</span> ~/.<span class="keyword">profile</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>即可完成环境变量的添加。</p>
                  <h2 id="6-验证安装"><a href="#6-验证安装" class="headerlink" title="6. 验证安装"></a>6. 验证安装</h2>
                  <p>配置完成后，就可以在命令行下直接执行<code>chromedriver</code>命令了：</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">chromedriver</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>如果输入控制台有类似图1-17所示的输出，则证明ChromeDriver的环境变量配置好了。</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/1-17.jpg" alt=""></p>
                  <p>图1-17 控制台输出</p>
                  <p>随后再在程序中测试，执行如下Python代码：</p>
                  <figure class="highlight armasm">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="symbol">from</span> <span class="keyword">selenium </span><span class="meta">import</span> webdriver</span><br><span class="line"><span class="keyword">browser </span>= webdriver.Chrome()</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行之后，如果弹出一个空白的Chrome浏览器，则证明所有的配置都没有问题。如果没有弹出，请检查之前的每一步配置。</p>
                  <p>如果弹出后闪退，则可能是ChromeDriver版本和Chrome版本不兼容，请更换ChromeDriver版本。</p>
                  <p>如果没有问题，接下来就可以利用Chrome来做网页抓取了。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-01-25 10:46:39" itemprop="dateCreated datePublished" datetime="2018-01-25T10:46:39+08:00">2018-01-25</time>
                </span>
                <span id="/5135.html" class="post-meta-item leancloud_visitors" data-flag-title="[Python3网络爬虫开发实战] 1.2.3-ChromeDriver的安装" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>1.8k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>2 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/5141.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a href="/5141.html" class="post-title-link" itemprop="url">[Python3网络爬虫开发实战] 1.2.2-Selenium的安装</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>Selenium是一个自动化测试工具，利用它我们可以驱动浏览器执行特定的动作，如点击、下拉等操作。对于一些JavaScript渲染的页面来说，这种抓取方式非常有效。下面我们来看看Selenium的安装过程。</p>
                  <h2 id="1-相关链接"><a href="#1-相关链接" class="headerlink" title="1. 相关链接"></a>1. 相关链接</h2>
                  <ul>
                    <li>官方网站：<a href="http://www.seleniumhq.org/" target="_blank" rel="noopener">http://www.seleniumhq.org</a></li>
                    <li>GitHub：<a href="https://github.com/SeleniumHQ/selenium/tree/master/py" target="_blank" rel="noopener">https://github.com/SeleniumHQ/selenium/tree/master/py</a></li>
                    <li>PyPI：<a href="https://pypi.python.org/pypi/selenium" target="_blank" rel="noopener">https://pypi.python.org/pypi/selenium</a></li>
                    <li>官方文档：<a href="http://selenium-python.readthedocs.io/" target="_blank" rel="noopener">http://selenium-python.readthedocs.io</a></li>
                    <li>中文文档：<a href="http://selenium-python-zh.readthedocs.io/" target="_blank" rel="noopener">http://selenium-python-zh.readthedocs.io</a></li>
                  </ul>
                  <h2 id="2-pip安装"><a href="#2-pip安装" class="headerlink" title="2. pip安装"></a>2. pip安装</h2>
                  <p>这里推荐直接使用pip安装，执行如下命令即可：</p>
                  <figure class="highlight cmake">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">pip3 <span class="keyword">install</span> selenium</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h2 id="3-wheel安装"><a href="#3-wheel安装" class="headerlink" title="3. wheel安装"></a>3. wheel安装</h2>
                  <p>此外，也可以到PyPI下载对应的wheel文件进行安装（下载地址：<a href="https://pypi.python.org/pypi/selenium/#downloads" target="_blank" rel="noopener">https://pypi.python.org/pypi/selenium/#downloads</a>），如最新版本为3.4.3，则下载selenium-3.4.3-py2.py3-none-any.whl即可。</p>
                  <p>然后进入wheel文件目录，使用pip安装：</p>
                  <figure class="highlight css">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="selector-tag">pip3</span> <span class="selector-tag">install</span> <span class="selector-tag">selenium-3</span><span class="selector-class">.4</span><span class="selector-class">.3-py2</span><span class="selector-class">.py3-none-any</span><span class="selector-class">.whl</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h2 id="4-验证安装"><a href="#4-验证安装" class="headerlink" title="4. 验证安装"></a>4. 验证安装</h2>
                  <p>进入Python命令行交互模式，导入Selenium包，如果没有报错，则证明安装成功：</p>
                  <figure class="highlight ruby">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">$ python3</span><br><span class="line"><span class="meta">&gt;&gt;</span>&gt; import selenium</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>但这样做还不够，因为我们还需要用浏览器（如Chrome、Firefox等）来配合Selenium工作。</p>
                  <p>后面我们会介绍Chrome、Firefox、PhantomJS三种浏览器的配置方式。有了浏览器，我们才可以配合Selenium进行页面的抓取。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-01-25 10:07:20" itemprop="dateCreated datePublished" datetime="2018-01-25T10:07:20+08:00">2018-01-25</time>
                </span>
                <span id="/5141.html" class="post-meta-item leancloud_visitors" data-flag-title="[Python3网络爬虫开发实战] 1.2.2-Selenium的安装" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>799</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>1 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/5132.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/5132.html" class="post-title-link" itemprop="url">[Python3网络爬虫开发实战] 1.2.1-Requests的安装</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>由于Requests属于第三方库，也就是Python默认不会自带这个库，所以需要我们手动安装。下面我们首先看一下它的安装过程。</p>
                  <h2 id="1-相关链接"><a href="#1-相关链接" class="headerlink" title="1. 相关链接"></a>1. 相关链接</h2>
                  <ul>
                    <li>GitHub：<a href="https://github.com/requests/requests" target="_blank" rel="noopener">https://github.com/requests/requests</a></li>
                    <li>PyPI：<a href="https://pypi.python.org/pypi/requests" target="_blank" rel="noopener">https://pypi.python.org/pypi/requests</a></li>
                    <li>官方文档：<a href="http://www.python-requests.org/" target="_blank" rel="noopener">http://www.python-requests.org</a></li>
                    <li>中文文档：<a href="http://docs.python-requests.org/zh_CN/latest" target="_blank" rel="noopener">http://docs.python-requests.org/zh_CN/latest</a></li>
                  </ul>
                  <h2 id="2-pip安装"><a href="#2-pip安装" class="headerlink" title="2. pip安装"></a>2. pip安装</h2>
                  <p>无论是Windows、Linux还是Mac，都可以通过pip这个包管理工具来安装。</p>
                  <p>在命令行界面中运行如下命令，即可完成Requests库的安装：</p>
                  <figure class="highlight cmake">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">pip3 <span class="keyword">install</span> requests</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这是最简单的安装方式，推荐使用这种方法安装。</p>
                  <h2 id="3-wheel安装"><a href="#3-wheel安装" class="headerlink" title="3. wheel安装"></a>3. wheel安装</h2>
                  <p>wheel是Python的一种安装包，其后缀为.whl，在网速较差的情况下可以选择下载wheel文件再安装，然后直接用<code>pip3</code>命令加文件名安装即可。</p>
                  <p>不过在这之前需要先安装wheel库，安装命令如下：</p>
                  <figure class="highlight cmake">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">pip3 <span class="keyword">install</span> wheel</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>然后到PyPI上下载对应的wheel文件，如最新版本为2.17.3，则打开<a href="https://pypi.python.org/pypi/requests/2.17.3#downloads" target="_blank" rel="noopener">https://pypi.python.org/pypi/requests/2.17.3#downloads</a>，下载requests-2.17.3-py2.py3-none-any.whl到本地。</p>
                  <p>随后在命令行界面进入wheel文件目录，利用<code>pip</code>安装即可：</p>
                  <figure class="highlight css">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="selector-tag">pip3</span> <span class="selector-tag">install</span> <span class="selector-tag">requests-2</span><span class="selector-class">.17</span><span class="selector-class">.3-py2</span><span class="selector-class">.py3-none-any</span><span class="selector-class">.whl</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样我们也可以完成Requests的安装。</p>
                  <h2 id="4-源码安装"><a href="#4-源码安装" class="headerlink" title="4. 源码安装"></a>4. 源码安装</h2>
                  <p>如果你不想用pip来安装，或者想获取某一特定版本，可以选择下载源码安装。</p>
                  <p>此种方式需要先找到此库的源码地址，然后下载下来再用命令安装。</p>
                  <p>Requests项目的地址是：<a href="https://github.com/kennethreitz/requests" target="_blank" rel="noopener">https://github.com/kennethreitz/requests</a>。</p>
                  <p>可以通过Git来下载源代码：</p>
                  <figure class="highlight crmsh">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">git <span class="keyword">clone</span> <span class="title">git</span>://github.com/kennethreitz/requests.git</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>或通过<code>curl</code>下载：</p>
                  <figure class="highlight awk">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">curl -OL https:<span class="regexp">//gi</span>thub.com<span class="regexp">/kennethreitz/</span>requests<span class="regexp">/tarball/m</span>aster</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>下载下来之后，进入目录，执行如下命令即可安装：</p>
                  <figure class="highlight vim">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">cd</span> requests</span><br><span class="line"><span class="keyword">python3</span> setup.<span class="keyword">py</span> install</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>命令执行结束后即可完成Requests的安装。由于这种安装方式比较烦琐，后面不再赘述。</p>
                  <h2 id="5-验证安装"><a href="#5-验证安装" class="headerlink" title="5. 验证安装"></a>5. 验证安装</h2>
                  <p>为了验证库是否已经安装成功，可以在命令行模式测试一下：</p>
                  <figure class="highlight ruby">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">$ python3</span><br><span class="line"><span class="meta">&gt;&gt;</span>&gt; import requests</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>首先输入<code>python3</code>，进入命令行模式，然后输入上述内容，如果什么错误提示也没有，就证明已经成功安装了Requests。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-01-25 09:50:40" itemprop="dateCreated datePublished" datetime="2018-01-25T09:50:40+08:00">2018-01-25</time>
                </span>
                <span id="/5132.html" class="post-meta-item leancloud_visitors" data-flag-title="[Python3网络爬虫开发实战] 1.2.1-Requests的安装" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>1.2k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>1 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/5081.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/5081.html" class="post-title-link" itemprop="url">[Python3网络爬虫开发实战] 1.2-请求库的安装</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>爬虫可以简单分为几步：抓取页面、分析页面和存储数据。</p>
                  <p>在抓取页面的过程中，我们需要模拟浏览器向服务器发出请求，所以需要用到一些Python库来实现HTTP请求操作。在本书中，我们用到的第三方库有Requests、Selenium和aiohttp等。</p>
                  <p>在本节中，我们介绍一下这些请求库的安装方法。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-01-25 09:40:19" itemprop="dateCreated datePublished" datetime="2018-01-25T09:40:19+08:00">2018-01-25</time>
                </span>
                <span id="/5081.html" class="post-meta-item leancloud_visitors" data-flag-title="[Python3网络爬虫开发实战] 1.2-请求库的安装" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>145</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>1 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/5059.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/5059.html" class="post-title-link" itemprop="url">[Python3网络爬虫开发实战] 1.1-Python3的安装</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>既然要用Python 3开发爬虫，那么第一步一定是安装Python 3。这里会介绍Windows、Linux和Mac三大平台下的安装过程。</p>
                  <h2 id="1-相关链接"><a href="#1-相关链接" class="headerlink" title="1. 相关链接"></a>1. 相关链接</h2>
                  <ul>
                    <li>官方网站：<a href="http://python.org/" target="_blank" rel="noopener">http://python.org</a></li>
                    <li>下载地址：<a href="https://www.python.org/downloads" target="_blank" rel="noopener">https://www.python.org/downloads</a></li>
                    <li>第三方库：<a href="https://pypi.python.org/pypi" target="_blank" rel="noopener">https://pypi.python.org/pypi</a></li>
                    <li>官方文档：<a href="https://docs.python.org/3" target="_blank" rel="noopener">https://docs.python.org/3</a></li>
                    <li>中文教程：<a href="http://www.runoob.com/python3/python3-tutorial.html" target="_blank" rel="noopener">http://www.runoob.com/python3/python3-tutorial.html</a></li>
                    <li>Awesome Python：<a href="https://github.com/vinta/awesome-python" target="_blank" rel="noopener">https://github.com/vinta/awesome-python</a></li>
                    <li>Awesome Python中文版：<a href="https://github.com/jobbole/awesome-python-cn" target="_blank" rel="noopener">https://github.com/jobbole/awesome-python-cn</a></li>
                  </ul>
                  <h2 id="2-Windows下的安装"><a href="#2-Windows下的安装" class="headerlink" title="2. Windows下的安装"></a>2. Windows下的安装</h2>
                  <p>在Windows下安装Python 3的方式有两种。</p>
                  <ul>
                    <li>一种是通过Anaconda安装，它提供了Python的科学计算环境，里面自带了Python以及常用的库。如果选用了这种方式，后面的环境配置方式会更加简便。</li>
                    <li>另一种是直接下载安装包安装，即标准的安装方式。</li>
                  </ul>
                  <p>下面我们依次介绍这两种安装方式，任选其一即可。</p>
                  <h3 id="1-Anaconda安装"><a href="#1-Anaconda安装" class="headerlink" title="(1) Anaconda安装"></a>(1) Anaconda安装</h3>
                  <p>Anaconda的官方下载链接为<a href="https://www.continuum.io/downloads" target="_blank" rel="noopener">https://www.continuum.io/downloads</a>，选择Python 3版本的安装包下载即可，如图1-1所示。</p>
                  <p><img src="http://epub.ituring.com.cn/api/storage/getbykey/screenshow?key=1708002eb50fe93aef25" alt="图像说明文字"></p>
                  <p>图1-1 Anaconda Windows下载页面</p>
                  <p>如果下载速度过慢，可以选择使用清华大学镜像，下载列表链接为<a href="https://mirrors.tuna.tsinghua.edu.cn/anaconda/archive/" target="_blank" rel="noopener">https://mirrors.tuna.tsinghua.edu.cn/anaconda/archive/</a>，使用说明链接为<a href="https://mirrors.tuna.tsinghua.edu.cn/help/anaconda/" target="_blank" rel="noopener">https://mirrors.tuna.tsinghua.edu.cn/help/anaconda/</a>。</p>
                  <p>下载完成之后，直接双击安装包安装即可。安装完成之后，Python 3的环境就配置好了。</p>
                  <h3 id="2-安装包安装"><a href="#2-安装包安装" class="headerlink" title="(2) 安装包安装"></a>(2) 安装包安装</h3>
                  <p>我们推荐直接下载安装包来安装，此时可以直接到官方网站下载Python 3的安装包：<a href="https://www.python.org/downloads/" target="_blank" rel="noopener">https://www.python.org/downloads/</a>。</p>
                  <p>写书时，Python的最新版本1是3.6.2，其下载链接为<a href="https://www.python.org/downloads/release/python-362/" target="_blank" rel="noopener">https://www.python.org/downloads/release/python-362/</a>，下载页面如图1-2所示。需要说明的是，实际的Python最新版本以官网为准。</p>
                  <p><img src="http://epub.ituring.com.cn/api/storage/getbykey/screenshow?key=170831c80abe71970c38" alt="图像说明文字"></p>
                  <p>图1-2 Python下载页面</p>
                  <blockquote>
                    <ol>
                      <li>若无特别说明，书中的最新版本均为作者写书时的情况，后面不再一一说明。</li>
                    </ol>
                  </blockquote>
                  <p>64位系统可以下载Windows x86-64 executable installer，32位系统可以下载Windows x86 executable installer。</p>
                  <p>下载完成之后，直接双击Python安装包，然后通过图形界面安装，接着设置Python的安装路径，完成后将Python 3和Python 3的Scripts目录配置到环境变量即可。</p>
                  <p>关于环境变量的配置，此处以Windows 10系统为例进行演示。</p>
                  <p>假如安装后的Python 3路径为C:\Python36，从资源管理器中打开该路径，如图1-3所示。</p>
                  <p><img src="http://epub.ituring.com.cn/api/storage/getbykey/screenshow?key=17082965ee593cae2365" alt="图像说明文字"></p>
                  <p>图1-3 Python安装目录</p>
                  <p>将该路径复制下来。</p>
                  <p>随后，右击“计算机”，从中选择“属性”，此时将打开系统属性窗口，如图1-4所示。</p>
                  <p><img src="http://epub.ituring.com.cn/api/storage/getbykey/screenshow?key=1708cdd34354955d1fde" alt="图像说明文字"></p>
                  <p>图1-4 系统属性</p>
                  <p>点击左侧的“高级系统设置”，即可看到在弹出的对话框下方看到“环境变量”按钮，如图1-5所示。</p>
                  <p><img src="http://epub.ituring.com.cn/api/storage/getbykey/screenshow?key=1708d35844164e63b26a" alt="图像说明文字"></p>
                  <p>图1-5 高级系统设置</p>
                  <p>点击“环境变量”按钮，找到系统变量下的Path变量，随后点击“编辑”按钮，如图1-6所示。</p>
                  <p><img src="http://epub.ituring.com.cn/api/storage/getbykey/screenshow?key=1708e0b523729330b0e8" alt="图像说明文字"></p>
                  <p>图1-6 环境变量</p>
                  <p>随后点击“新建”，新建一个条目，将刚才复制的C:\Python36复制进去。这里需要说明的是，此处的路径就是你的Python 3安装目录，请自行替换。然后，再把C:\Python36\Scripts路径复制进去，如图1-7所示。</p>
                  <p><img src="http://epub.ituring.com.cn/api/storage/getbykey/screenshow?key=1708d8f0669ffcf518a7" alt="图像说明文字"></p>
                  <p>图1-7 编辑环境变量</p>
                  <p>最后，点击“确定”按钮即可完成环境变量的配置。</p>
                  <p>配置好环境变量后，我们就可以在命令行中直接执行环境变量路径下的可执行文件了，如<code>python</code>、<code>pip</code>等命令。</p>
                  <h3 id="3-添加别名"><a href="#3-添加别名" class="headerlink" title="(3) 添加别名"></a>(3) 添加别名</h3>
                  <p>上面这两种安装方式任选其一即可完成安装，但如果之前安装过Python 2的话，可能会导致版本冲突问题，比如在命令行下输入<code>python</code>就不知道是调用的Python 2还是Python 3了。为了解决这个问题，建议将安装目录中的python.exe复制一份，命名为python3.exe，这样便可以调用<code>python3</code>命令了。实际上，它和<code>python</code>命令是完全一致的，这样只是为了可以更好地区分Python版本。当然，如果没有安装过Python 2的话，也建议添加此别名，添加完毕之后的效果如图1-8所示。</p>
                  <p><img src="http://epub.ituring.com.cn/api/storage/getbykey/screenshow?key=17089ea4c3d0be341d8e" alt="图像说明文字"></p>
                  <p>图1-8 添加别名</p>
                  <p>对于pip来说，安装包中自带了pip3.exe可执行文件，我们也可以直接使用<code>pip3</code>命令，无需额外配置。</p>
                  <h3 id="4-测试验证"><a href="#4-测试验证" class="headerlink" title="(4) 测试验证"></a>(4) 测试验证</h3>
                  <p>安装完成后，可以通过命令行测试一下安装是否成功。在“开始”菜单中搜索<code>cmd</code>，找到命令提示符，此时就进入命令行模式了。输入<code>python</code>，测试一下能否成功调用Python。如果添加了别名的话，可以输入<code>python3</code>测试，这里输入的是<code>python3</code>，测试结果如图1-9所示。</p>
                  <p><img src="http://epub.ituring.com.cn/api/storage/getbykey/screenshow?key=17089590fa9f68124670" alt="图像说明文字"></p>
                  <p>图1-9 测试验证页面</p>
                  <p>输出结果类似如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">$ python3</span><br><span class="line">Python 3.6.1 (v3.6.1:69c0db5, Mar 21 2017, 17:54:52) [MSC v.1900 32 bit (Intel)] on win32</span><br><span class="line">Type <span class="string">"help"</span>, <span class="string">"copyright"</span>, <span class="string">"credits"</span> <span class="keyword">or</span> <span class="string">"license"</span> <span class="keyword">for</span> more information.</span><br><span class="line">&gt;&gt;&gt; <span class="builtin-name">print</span>(<span class="string">'Hello World'</span>)</span><br><span class="line">Hello World</span><br><span class="line">&gt;&gt;&gt; exit()</span><br><span class="line">$ pip3 -V</span><br><span class="line">pip 9.0.1 <span class="keyword">from</span> c:\python36\lib\site-packages (python 3.6)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>如果出现了类似上面的提示，则证明Python 3和pip 3均安装成功；如果提示命令不存在，那么请检查下环境变量的配置情况。</p>
                  <h2 id="3-Linux下的安装"><a href="#3-Linux下的安装" class="headerlink" title="3. Linux下的安装"></a>3. Linux下的安装</h2>
                  <p>Linux下的安装方式有多种：命令安装、源码安装和Anaconda安装。</p>
                  <p>使用源码安装需要自行编译，时间较长。推荐使用系统自带的命令或Anaconda安装，简单、高效。这里分别讲解这3种安装方式。</p>
                  <h3 id="1-命令行安装"><a href="#1-命令行安装" class="headerlink" title="(1) 命令行安装"></a>(1) 命令行安装</h3>
                  <p>不同的Linux发行版本的安装方式又有不同，在此分别予以介绍。</p>
                  <h4 id="CentOS、Red-Hat"><a href="#CentOS、Red-Hat" class="headerlink" title="CentOS、Red Hat"></a>CentOS、Red Hat</h4>
                  <p>如果是CentOS或Red Hat版本，则使用<code>yum</code>命令安装即可。</p>
                  <p>下面列出了Python 3.5和Python 3.4两个版本的安装方法，可以自行选择。</p>
                  <p>Python 3.5版本：</p>
                  <figure class="highlight gml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">sudo yum install -<span class="symbol">y</span> https:<span class="comment">//centos7.iuscommunity.org/ius-release.rpm</span></span><br><span class="line">sudo yum update</span><br><span class="line">sudo yum install -<span class="symbol">y</span> python35u python35u-libs python35u-devel python35u-pip</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>执行完毕后，便可以成功安装Python 3.5及pip 3了。</p>
                  <p>Python 3.4版本：</p>
                  <figure class="highlight properties">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">sudo</span> <span class="string">yum groupinstall -y development tools</span></span><br><span class="line"><span class="attr">sudo</span> <span class="string">yum install -y epel-release python34-devel  libxslt-devel libxml2-devel openssl-devel</span></span><br><span class="line"><span class="attr">sudo</span> <span class="string">yum install -y python34</span></span><br><span class="line"><span class="attr">sudo</span> <span class="string">yum install -y python34-setuptools</span></span><br><span class="line"><span class="attr">sudo</span> <span class="string">easy_install-3.4 pip</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>执行完毕后，便可以成功安装Python 3.4及pip 3了。</p>
                  <h4 id="Ubuntu、Debian和Deepin"><a href="#Ubuntu、Debian和Deepin" class="headerlink" title="Ubuntu、Debian和Deepin"></a>Ubuntu、Debian和Deepin</h4>
                  <p>首先安装Python 3，这里使用<code>apt-get</code>安装即可。在安装前，还需安装一些基础库，相关命令如下：</p>
                  <figure class="highlight q">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">sudo apt-<span class="built_in">get</span> install -y python3-<span class="built_in">dev</span> build-essential libssl-<span class="built_in">dev</span> libffi-<span class="built_in">dev</span> libxml2 libxml2-<span class="built_in">dev</span> libxslt1-<span class="built_in">dev</span> zlib1g-<span class="built_in">dev</span> libcurl4-openssl-<span class="built_in">dev</span></span><br><span class="line">sudo apt-<span class="built_in">get</span> install -y python3</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>执行完上述命令后，就可以成功安装Python 3了。</p>
                  <p>然后还需要安装pip 3，这里仍然使用<code>apt-get</code>安装即可，相关命令如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">sudo apt-<span class="builtin-name">get</span> install -y python3-pip</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>执行完毕后，便可以成功安装Python 3及pip 3了。</p>
                  <h3 id="2-源码安装"><a href="#2-源码安装" class="headerlink" title="(2) 源码安装"></a>(2) 源码安装</h3>
                  <p>如果命令行的安装方式有问题，还可以下载Python 3源码进行安装。</p>
                  <p>源码下载地址为<a href="https://www.python.org/ftp/python/" target="_blank" rel="noopener">https://www.python.org/ftp/python/</a>，可以自行选用想要的版本进行安装。这里以Python 3.6.2为例进行说明，安装路径设置为/usr/local/python3。</p>
                  <p>首先，创建安装目录，相关命令如下：</p>
                  <figure class="highlight maxima">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">sudo <span class="built_in">mkdir</span> /usr/<span class="built_in">local</span>/python3</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>随后下载安装包并解压进入，相关命令如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">wget --no-check-certificate https:<span class="comment">//www.python.org/ftp/python/3.6.2/Python-3.6.2.tgz</span></span><br><span class="line">tar -xzvf Python<span class="number">-3.6</span><span class="number">.2</span>.tgz</span><br><span class="line">cd Python<span class="number">-3.6</span><span class="number">.2</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>接下来，编译安装。所需的时间可能较长，请耐心等待，命令如下：</p>
                  <figure class="highlight gauss">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">sudo ./configure --prefix=/usr/<span class="keyword">local</span>/python3</span><br><span class="line">sudo <span class="built_in">make</span></span><br><span class="line">sudo <span class="built_in">make</span> install</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>安装完成之后，创建Python 3链接，相关命令如下：</p>
                  <figure class="highlight awk">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">sudo ln -s <span class="regexp">/usr/</span>local<span class="regexp">/python3/</span>bin<span class="regexp">/python3 /u</span>sr<span class="regexp">/bin/</span>python3</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>随后下载pip安装包并安装，命令如下：</p>
                  <figure class="highlight vim">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">wget --<span class="keyword">no</span>-check-certificate http<span class="variable">s:</span>//github.<span class="keyword">com</span>/pypa/pip/archive/<span class="number">9.0</span>.<span class="number">1</span>.tar.gz</span><br><span class="line">tar -xzvf <span class="number">9.0</span>.<span class="number">1</span>.tar.gz</span><br><span class="line"><span class="keyword">cd</span> pip-<span class="number">9.0</span>.<span class="number">1</span></span><br><span class="line"><span class="keyword">python3</span> setup.<span class="keyword">py</span> install</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>安装完成后再创建pip 3链接，相关命令如下：</p>
                  <figure class="highlight awk">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">sudo ln -s <span class="regexp">/usr/</span>local<span class="regexp">/python3/</span>bin<span class="regexp">/pip /u</span>sr<span class="regexp">/bin/</span>pip3</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样就成功安装好了Python 3及pip 3。</p>
                  <h3 id="3-Anaconda安装"><a href="#3-Anaconda安装" class="headerlink" title="(3) Anaconda安装"></a>(3) Anaconda安装</h3>
                  <p>Anaconda同样支持Linux，其官方下载链接为<a href="http://epub.ituring.com.cn/article/edit/[https://www.continuum.io/downloads" target="_blank" rel="noopener">https://www.continuum.io/downloads</a>，选择Python 3版本的安装包下载即可，如图1-10所示。</p>
                  <p><img src="http://epub.ituring.com.cn/api/storage/getbykey/screenshow?key=1708a13922b218a1bbb2" alt="图像说明文字"></p>
                  <p>图1-10 Anaconda Linux下载页面</p>
                  <p>如果下载速度过慢，同样可以使用清华镜像，具体可参考Windows部分的介绍，在此不再赘述。</p>
                  <h3 id="4-测试验证-1"><a href="#4-测试验证-1" class="headerlink" title="(4) 测试验证"></a>(4) 测试验证</h3>
                  <p>在命令行界面下测试Python 3和pip 3是否安装成功：</p>
                  <figure class="highlight awk">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">$ python3</span><br><span class="line">Python <span class="number">3.5</span>.<span class="number">2</span> (default, Nov <span class="number">17</span> <span class="number">2016</span>, <span class="number">17</span>:<span class="number">05</span>:<span class="number">23</span>) </span><br><span class="line">Type <span class="string">"help"</span>, <span class="string">"copyright"</span>, <span class="string">"credits"</span> or <span class="string">"license"</span> <span class="keyword">for</span> more information.</span><br><span class="line">&gt;&gt;&gt; <span class="keyword">exit</span>()</span><br><span class="line">$ pip3 -V</span><br><span class="line">pip <span class="number">8.1</span>.<span class="number">1</span> from <span class="regexp">/usr/</span>lib<span class="regexp">/python3/</span>dist-packages (python <span class="number">3.5</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>若出现类似上面的提示，则证明Python 3和pip 3安装成功。</p>
                  <h2 id="4-Mac下的安装"><a href="#4-Mac下的安装" class="headerlink" title="4. Mac下的安装"></a>4. Mac下的安装</h2>
                  <p>在Mac下同样有多种安装方式，如Homebrew、安装包安装、Anaconda安装等，这里推荐使用Homebrew安装。</p>
                  <h3 id="1-Homebrew安装"><a href="#1-Homebrew安装" class="headerlink" title="(1) Homebrew安装"></a>(1) Homebrew安装</h3>
                  <p>Homebrew是Mac平台下强大的包管理工具，其官方网站是<a href="https://brew.sh/" target="_blank" rel="noopener">https://brew.sh/</a>。</p>
                  <p>执行如下命令，即可安装Homebrew：</p>
                  <figure class="highlight sql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/<span class="keyword">install</span>/<span class="keyword">master</span>/<span class="keyword">install</span>)<span class="string">"</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>安装完成后，便可以使用<code>brew</code>命令安装Python 3和pip 3了：</p>
                  <figure class="highlight mipsasm">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">brew </span><span class="keyword">install </span>python3</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>命令执行完成后，我们发现Python 3和pip 3均已成功安装。</p>
                  <h3 id="2-安装包安装-1"><a href="#2-安装包安装-1" class="headerlink" title="(2) 安装包安装"></a>(2) 安装包安装</h3>
                  <p>可以到官方网站下载Python 3安装包。链接为<a href="https://www.python.org/downloads/" target="_blank" rel="noopener">https://www.python.org/downloads/</a>，页面如图1-2所示。</p>
                  <p>在Mac平台下，可以选择下载Mac OS X 64-bit/32-bit installer，下载完成后，打开安装包按照提示安装即可。</p>
                  <h3 id="3-Anaconda安装-1"><a href="#3-Anaconda安装-1" class="headerlink" title="(3) Anaconda安装"></a>(3) Anaconda安装</h3>
                  <p>Anaconda同样支持Mac，其官方下载链接为：<a href="http://epub.ituring.com.cn/article/edit/[https://www.continuum.io/downloads" target="_blank" rel="noopener">https://www.continuum.io/downloads</a>，选择Python 3版本的安装包下载即可，如图1-11所示。</p>
                  <p><img src="http://epub.ituring.com.cn/api/storage/getbykey/screenshow?key=17089b034fd975411c37" alt="图像说明文字"></p>
                  <p>图1-11 Anaconda Mac下载页面</p>
                  <p>如果下载速度过慢，同样可以使用清华镜像，具体可参考Windows部分的介绍，在此不再赘述。</p>
                  <h3 id="4-测试验证-2"><a href="#4-测试验证-2" class="headerlink" title="(4) 测试验证"></a>(4) 测试验证</h3>
                  <p>打开终端，在命令行界面中测试Python 3和pip 3是否成功安装，如图1-12所示。</p>
                  <p><img src="http://epub.ituring.com.cn/api/storage/getbykey/screenshow?key=1708788341be5f65d01e" alt="图像说明文字"></p>
                  <p>图1-12 测试验证页面</p>
                  <p>若出现上面的提示，则证明Python 3和pip 3安装成功。</p>
                  <p>本节中，我们介绍了3大平台Windows、Linux和Mac下Python 3的安装方式。安装完成后，我们便可以开启Python爬虫的征程了。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-01-25 09:30:39" itemprop="dateCreated datePublished" datetime="2018-01-25T09:30:39+08:00">2018-01-25</time>
                </span>
                <span id="/5059.html" class="post-meta-item leancloud_visitors" data-flag-title="[Python3网络爬虫开发实战] 1.1-Python3的安装" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>5.5k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>5 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/5054.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/5054.html" class="post-title-link" itemprop="url">[Python3网络爬虫开发实战] 1-开发环境配置</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>工欲善其事，必先利其器！</p>
                  <p>编写和运行程序之前我们必须要先把开发环境配置好，只有配置好了环境并且有了更方便的开发工具我们才能更加高效地用程序实现相应的功能，然而很多情况下我们可能在最开始就卡在环境配置上，如果这个过程花费了太多时间，想必学习的兴趣就下降了大半，所以本章专门来对本书中所有的环境配置做一下说明。</p>
                  <p>本章是本书使用的所有库及工具的安装过程讲解，为了使书的条理更加清晰，本书将环境配置的过程统一合并为一章，本章不必逐节阅读，可以在需要的时候进行查阅。</p>
                  <p>文中在介绍安装过程的时候会尽量兼顾各个平台，另外会将一些安装常见的错误指出，以便快速高效地搭建好编程环境。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-01-25 08:10:58" itemprop="dateCreated datePublished" datetime="2018-01-25T08:10:58+08:00">2018-01-25</time>
                </span>
                <span id="/5054.html" class="post-meta-item leancloud_visitors" data-flag-title="[Python3网络爬虫开发实战] 1-开发环境配置" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>279</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>1 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/5052.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/5052.html" class="post-title-link" itemprop="url">Python3网络爬虫开发实战教程</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>本内容来自于《Python3网络爬虫开发实战》一书。 书籍购买地址： <a href="https://item.jd.com/12333540.html" target="_blank" rel="noopener">https://item.jd.com/12333540.html</a></p>
                  <p>本书通过多个实战案例详细介绍了 Python3 网络爬虫的知识，本书由图灵教育-人民邮电出版社出版发行，版权所有，禁止转载。</p>
                  <ul>
                    <li><a href="https://cuiqingcai.com/5054.html">1-开发环境配置</a></li>
                    <li><a href="https://cuiqingcai.com/5059.html">1.1-Python3的安装</a></li>
                    <li><a href="https://cuiqingcai.com/5081.html" title="[Python3网络爬虫开发实战] 1.2-请求库的安装">1.2-请求库的安装</a></li>
                    <li><a href="https://cuiqingcai.com/5132.html" title="[Python3网络爬虫开发实战] 1.2.1-Requests的安装">1.2.1-Requests的安装</a></li>
                    <li><a href="https://cuiqingcai.com/5141.html" title="[Python3网络爬虫开发实战] 1.2.2-Selenium的安装">1.2.2-Selenium的安装</a></li>
                    <li><a href="https://cuiqingcai.com/5135.html" title="[Python3网络爬虫开发实战] 1.2.3-ChromeDriver的安装">1.2.3-ChromeDriver的安装</a></li>
                    <li><a href="https://cuiqingcai.com/5153.html" title="[Python3网络爬虫开发实战] 1.2.4-GeckoDriver的安装">1.2.4-GeckoDriver的安装</a></li>
                    <li><a href="https://cuiqingcai.com/5159.html" title="[Python3网络爬虫开发实战] 1.2.5-PhantomJS的安装">1.2.5-PhantomJS的安装</a></li>
                    <li><a href="https://cuiqingcai.com/5163.html" title="[Python3网络爬虫开发实战] 1.2.6-aiohttp的安装">1.2.6-aiohttp的安装</a></li>
                    <li><a href="https://cuiqingcai.com/5168.html" title="[Python3网络爬虫开发实战] 1.3-解析库的安装">1.3-解析库的安装</a></li>
                    <li><a href="https://cuiqingcai.com/5180.html" title="[Python3网络爬虫开发实战] 1.3.1-lxml的安装">1.3.1-lxml的安装</a></li>
                    <li><a href="https://cuiqingcai.com/5183.html" title="[Python3网络爬虫开发实战] 1.3.2-Beautiful Soup的安装">1.3.2-Beautiful Soup的安装</a></li>
                    <li><a href="https://cuiqingcai.com/5186.html" title="[Python3网络爬虫开发实战] 1.3.3-pyquery的安装">1.3.3-pyquery的安装</a></li>
                    <li><a href="https://cuiqingcai.com/5189.html" title="[Python3网络爬虫开发实战] 1.3.4-tesserocr的安装">1.3.4-tesserocr的安装</a></li>
                    <li><a href="https://cuiqingcai.com/5197.html" title="[Python3网络爬虫开发实战] 1.4-数据库的安装">1.4-数据库的安装</a></li>
                    <li><a href="https://cuiqingcai.com/5200.html" title="[Python3网络爬虫开发实战] 1.4.1-MySQL的安装">1.4.1-MySQL的安装</a></li>
                    <li><a href="https://cuiqingcai.com/5205.html" title="[Python3网络爬虫开发实战] 1.4.2-MongoDB安装">1.4.2-MongoDB安装</a></li>
                    <li><a href="https://cuiqingcai.com/5219.html" title="[Python3网络爬虫开发实战] 1.4.3-Redis的安装">1.4.3-Redis的安装</a></li>
                    <li><a href="https://cuiqingcai.com/5224.html" title="[Python3网络爬虫开发实战] 1.5-存储库的安装">1.5-存储库的安装</a></li>
                    <li><a href="https://cuiqingcai.com/5227.html" title="[Python3网络爬虫开发实战] 1.5.1-PyMySQL的安装">1.5.1-PyMySQL的安装</a></li>
                    <li><a href="https://cuiqingcai.com/5230.html" title="[Python3网络爬虫开发实战] 1.5.2-PyMongo的安装">1.5.2-PyMongo的安装</a></li>
                    <li><a href="https://cuiqingcai.com/5233.html" title="[Python3网络爬虫开发实战] 1.5.3-redis-py的安装">1.5.3-redis-py的安装</a></li>
                    <li><a href="https://cuiqingcai.com/5236.html" title="[Python3网络爬虫开发实战] 1.5.4-RedisDump的安装">1.5.4-RedisDump的安装</a></li>
                    <li><a href="https://cuiqingcai.com/5239.html" title="[Python3网络爬虫开发实战] 1.6-Web库的安装">1.6-Web库的安装</a></li>
                    <li><a href="https://cuiqingcai.com/5244.html" title="[Python3网络爬虫开发实战] 1.6.1-Flask的安装">1.6.1-Flask的安装</a></li>
                    <li><a href="https://cuiqingcai.com/5248.html" title="[Python3网络爬虫开发实战] 1.6.2-Tornado的安装">1.6.2-Tornado的安装</a></li>
                    <li><a href="https://cuiqingcai.com/5252.html" title="[Python3网络爬虫开发实战] 1.7-App爬取相关库的安装">1.7-App爬取相关库的安装</a></li>
                    <li><a href="https://cuiqingcai.com/5255.html" title="[Python3网络爬虫开发实战] 1.7.1-Charles的安装">1.7.1-Charles的安装</a></li>
                    <li><a href="https://cuiqingcai.com/5391.html" title="[Python3网络爬虫开发实战] 1.7.2-mitmproxy的安装">1.7.2-mitmproxy的安装</a></li>
                    <li><a href="https://cuiqingcai.com/5407.html" title="[Python3网络爬虫开发实战] 1.7.3-Appium的安装">1.7.3-Appium的安装</a></li>
                    <li><a href="https://cuiqingcai.com/5413.html" title="[Python3网络爬虫开发实战] 1.8-爬虫框架的安装">1.8-爬虫框架的安装</a></li>
                    <li><a href="https://cuiqingcai.com/5416.html" title="[Python3网络爬虫开发实战] 1.8.1-pyspider的安装">1.8.1-pyspider的安装</a></li>
                    <li><a href="https://cuiqingcai.com/5421.html" title="[Python3网络爬虫开发实战] 1.8.2-Scrapy的安装">1.8.2-Scrapy的安装</a></li>
                    <li><a href="https://cuiqingcai.com/5428.html" title="[Python3网络爬虫开发实战] 1.8.3-Scrapy-Splash的安装">1.8.3-Scrapy-Splash的安装</a></li>
                    <li><a href="https://cuiqingcai.com/5432.html" title="[Python3网络爬虫开发实战] 1.8.4-Scrapy-Redis的安装">1.8.4-Scrapy-Redis的安装</a></li>
                    <li><a href="https://cuiqingcai.com/5435.html" title="[Python3网络爬虫开发实战] 1.9-部署相关库的安装">1.9-部署相关库的安装</a></li>
                    <li><a href="https://cuiqingcai.com/5438.html" title="[Python3网络爬虫开发实战] 1.9.1-Docker的安装">1.9.1-Docker的安装</a></li>
                    <li><a href="https://cuiqingcai.com/5445.html" title="[Python3网络爬虫开发实战] 1.9.2-Scrapyd的安装">1.9.2-Scrapyd的安装</a></li>
                    <li><a href="https://cuiqingcai.com/5449.html" title="[Python3网络爬虫开发实战] 1.9.3-Scrapyd-Client的安装">1.9.3-Scrapyd-Client的安装</a></li>
                    <li><a href="https://cuiqingcai.com/5453.html" title="[Python3网络爬虫开发实战] 1.9.4-Scrapyd API的安装">1.9.4-Scrapyd API的安装</a></li>
                    <li><a href="https://cuiqingcai.com/5456.html" title="[Python3网络爬虫开发实战] 1.9.5-Scrapyrt的安装">1.9.5-Scrapyrt的安装</a></li>
                    <li><a href="https://cuiqingcai.com/5459.html" title="[Python3网络爬虫开发实战] 1.9.6-Gerapy的安装">1.9.6-Gerapy的安装</a></li>
                    <li><a href="https://cuiqingcai.com/5462.html" title="[Python3网络爬虫开发实战] 2-爬虫基础">2-爬虫基础</a></li>
                    <li><a href="https://cuiqingcai.com/5465.html" title="[Python3网络爬虫开发实战] 2.1-HTTP基本原理">2.1-HTTP基本原理</a></li>
                    <li><a href="https://cuiqingcai.com/5476.html" title="[Python3网络爬虫开发实战] 2.2-网页基础">2.2-网页基础</a></li>
                    <li><a href="https://cuiqingcai.com/5484.html" title="[Python3网络爬虫开发实战] 2.3-爬虫的基本原理">2.3-爬虫的基本原理</a></li>
                    <li><a href="https://cuiqingcai.com/5487.html" title="[Python3网络爬虫开发实战] 2.4-会话和Cookies">2.4-会话和Cookies</a></li>
                    <li><a href="https://cuiqingcai.com/5491.html" title="[Python3网络爬虫开发实战] 2.5-代理的基本原理">2.5-代理的基本原理</a></li>
                    <li><a href="https://cuiqingcai.com/5494.html" title="[Python3网络爬虫开发实战] 3-基本库的使用">3-基本库的使用</a></li>
                    <li><a href="https://cuiqingcai.com/5497.html" title="[Python3网络爬虫开发实战] 3.1-使用urllib">3.1-使用urllib</a></li>
                    <li><a href="https://cuiqingcai.com/5500.html" title="[Python3网络爬虫开发实战] 3.1.1-发送请求">3.1.1-发送请求</a></li>
                    <li><a href="https://cuiqingcai.com/5505.html" title="[Python3网络爬虫开发实战] 3.1.2-处理异常">3.1.2-处理异常</a></li>
                    <li><a href="https://cuiqingcai.com/5508.html" title="[Python3网络爬虫开发实战] 3.1.3-解析链接">3.1.3-解析链接</a></li>
                    <li><a href="https://cuiqingcai.com/5511.html" title="[Python3网络爬虫开发实战] 3.1.4-分析Robots协议">3.1.4-分析Robots协议</a></li>
                    <li><a href="https://cuiqingcai.com/5514.html" title="[Python3网络爬虫开发实战] 3.2-使用requests">3.2-使用requests</a></li>
                    <li><a href="https://cuiqingcai.com/5517.html" title="[Python3网络爬虫开发实战] 3.2.1-基本用法">3.2.1-基本用法</a></li>
                    <li><a href="https://cuiqingcai.com/5523.html" title="[Python3网络爬虫开发实战] 3.2.2-高级用法">3.2.2-高级用法</a></li>
                    <li><a href="https://cuiqingcai.com/5530.html" title="[Python3网络爬虫开发实战] 3.3-正则表达式">3.3-正则表达式</a></li>
                    <li><a href="https://cuiqingcai.com/5534.html" title="[Python3网络爬虫开发实战] 3.4-抓取猫眼电影排行">3.4-抓取猫眼电影排行</a></li>
                    <li><a href="https://cuiqingcai.com/5542.html" title="[Python3网络爬虫开发实战] 4-解析库的使用">4-解析库的使用</a></li>
                    <li><a href="https://cuiqingcai.com/5545.html" title="[Python3网络爬虫开发实战] 4.1-使用XPath">4.1-使用XPath</a></li>
                    <li><a href="https://cuiqingcai.com/5548.html" title="[Python3网络爬虫开发实战] 4.2-使用Beautiful Soup">4.2-使用Beautiful Soup</a></li>
                    <li><a href="https://cuiqingcai.com/5551.html" title="[Python3网络爬虫开发实战] 4.3-使用pyquery">4.3-使用pyquery</a></li>
                    <li><a href="https://cuiqingcai.com/5554.html" title="[Python3网络爬虫开发实战] 5-数据存储">5-数据存储</a></li>
                    <li><a href="https://cuiqingcai.com/5557.html" title="[Python3网络爬虫开发实战] 5.1-文件存储">5.1-文件存储</a></li>
                    <li><a href="https://cuiqingcai.com/5560.html" title="[Python3网络爬虫开发实战] 5.1.1-TXT文本存储">5.1.1-TXT文本存储</a></li>
                    <li><a href="https://cuiqingcai.com/5564.html" title="[Python3网络爬虫开发实战] 5.1.2-JSON文件存储">5.1.2-JSON文件存储</a></li>
                    <li><a href="https://cuiqingcai.com/5571.html" title="[Python3网络爬虫开发实战] 5.1.3-CSV文件存储">5.1.3-CSV文件存储</a></li>
                    <li><a href="https://cuiqingcai.com/5575.html" title="[Python3网络爬虫开发实战] 5.2-关系型数据库存储">5.2-关系型数据库存储</a></li>
                    <li><a href="https://cuiqingcai.com/5578.html" title="[Python3网络爬虫开发实战] 5.2.1-MySQL存储">5.2.1-MySQL存储</a></li>
                    <li><a href="https://cuiqingcai.com/5581.html" title="[Python3网络爬虫开发实战] 5.3-非关系型数据库存储">5.3-非关系型数据库存储</a></li>
                    <li><a href="https://cuiqingcai.com/5584.html" title="[Python3网络爬虫开发实战] 5.3.1-MongoDB存储">5.3.1-MongoDB存储</a></li>
                    <li><a href="https://cuiqingcai.com/5587.html" title="[Python3网络爬虫开发实战] 5.3.2-Redis存储">5.3.2-Redis存储</a></li>
                    <li><a href="https://cuiqingcai.com/5590.html" title="[Python3网络爬虫开发实战] 6-Ajax数据爬取">6-Ajax数据爬取</a></li>
                    <li><a href="https://cuiqingcai.com/5593.html" title="[Python3网络爬虫开发实战] 6.1-什么是Ajax">6.1-什么是Ajax</a></li>
                    <li><a href="https://cuiqingcai.com/5597.html" title="[Python3网络爬虫开发实战] 6.2-Ajax分析方法">6.2-Ajax分析方法</a></li>
                    <li><a href="https://cuiqingcai.com/5609.html" title="[Python3网络爬虫开发实战] 6.3-Ajax结果提取">6.3-Ajax结果提取</a></li>
                    <li><a href="https://cuiqingcai.com/5616.html" title="[Python3网络爬虫开发实战] 6.4-分析Ajax爬取今日头条街拍美图">6.4-分析Ajax爬取今日头条街拍美图</a></li>
                    <li><a href="https://cuiqingcai.com/5627.html" title="[Python3网络爬虫开发实战] 7-动态渲染页面爬取">7-动态渲染页面爬取</a></li>
                    <li><a href="https://cuiqingcai.com/5630.html" title="[Python3网络爬虫开发实战] 7.1-Selenium的使用">7.1-Selenium的使用</a></li>
                    <li><a href="https://cuiqingcai.com/5638.html" title="[Python3网络爬虫开发实战] 7.2-Splash的使用">7.2-Splash的使用</a></li>
                    <li><a href="https://cuiqingcai.com/5654.html" title="[Python3网络爬虫开发实战] 7.3-Splash负载均衡配置">7.3-Splash负载均衡配置</a></li>
                    <li><a href="https://cuiqingcai.com/5657.html" title="[Python3网络爬虫开发实战] 7.4-使用Selenium爬取淘宝商品">7.4-使用Selenium爬取淘宝商品</a></li>
                    <li><a href="hhttps://cuiqingcai.com/7032.html">8-验证码的识别</a></li>
                    <li><a href="https://cuiqingcai.com/7035.html">8.1-图形验证码的识别</a></li>
                    <li><a href="https://cuiqingcai.com/7037.html">8.2-极验滑动验证码的识别</a></li>
                    <li><a href="https://cuiqingcai.com/7039.html">8.3-点触验证码的识别</a></li>
                    <li><a href="https://cuiqingcai.com/7041.html">8.4-微博宫格验证码的识别</a></li>
                    <li><a href="https://cuiqingcai.com/7043.html">9-代理的使用</a></li>
                    <li><a href="https://cuiqingcai.com/7045.html">9.1-代理的设置</a></li>
                    <li><a href="https://cuiqingcai.com/7048.html">9.2-代理池的维护</a></li>
                    <li><a href="https://cuiqingcai.com/7051.html">9.3-付费代理的使用</a></li>
                    <li><a href="https://cuiqingcai.com/8361.html">9.4-ADSL拨号代理</a></li>
                    <li><a href="https://cuiqingcai.com/7844.html">9.5-使用代理爬取微信公众号文章</a></li>
                    <li><a href="https://cuiqingcai.com/5678.html">10-模拟登录</a></li>
                    <li><a href="https://cuiqingcai.com/8229.html">10.1-模拟登录并爬取GitHub</a></li>
                    <li><a href="https://cuiqingcai.com/8243.html">10.2-Cookies池的搭建</a></li>
                    <li><a href="https://cuiqingcai.com/5678.html">11-App的爬取</a></li>
                    <li><a href="https://cuiqingcai.com/8247.html">11.1-Charles的使用</a></li>
                    <li><a href="https://cuiqingcai.com/8260.html">11.2-mitmproxy的使用</a></li>
                    <li><a href="https://cuiqingcai.com/8263.html">11.3-mitmdump爬取“得到”App电子书信息</a></li>
                    <li><a href="https://cuiqingcai.com/8290.html">11.4-Appium的基本使用</a></li>
                    <li><a href="https://cuiqingcai.com/8293.html">11.5-Appium爬取微信朋友圈</a></li>
                    <li><a href="https://cuiqingcai.com/8306.html">11.6-Appium+mitmdump爬取京东商品</a></li>
                    <li><a href="https://cuiqingcai.com/5678.html">12-pyspider框架的使用</a></li>
                    <li><a href="https://cuiqingcai.com/8309.html">12.1-pyspider框架介绍</a></li>
                    <li><a href="https://cuiqingcai.com/8317.html">12.2-pyspider的基本使用</a></li>
                    <li><a href="https://cuiqingcai.com/8320.html">12.3-pyspider用法详解</a></li>
                    <li><a href="https://cuiqingcai.com/5678.html">13-Scrapy框架的使用</a></li>
                    <li><a href="https://cuiqingcai.com/8364.html">13.1-Scrapy框架介绍</a></li>
                    <li><a href="https://cuiqingcai.com/8337.html">13.2-Scrapy入门</a></li>
                    <li><a href="https://cuiqingcai.com/8350.html">13.3-Selector的用法</a></li>
                    <li><a href="https://cuiqingcai.com/8353.html">13.4-Spider的用法</a></li>
                    <li><a href="https://cuiqingcai.com/5678.html">13.5-Downloader Middleware的用法</a></li>
                    <li><a href="https://cuiqingcai.com/5678.html">13.6-Spider Middleware的用法</a></li>
                    <li><a href="https://cuiqingcai.com/5678.html">13.7-Item Pipeline的用法</a></li>
                    <li><a href="https://cuiqingcai.com/5678.html">13.8-Scrapy对接Selenium</a></li>
                    <li><a href="https://cuiqingcai.com/5678.html">13.9-Scrapy对接Splash</a></li>
                    <li><a href="https://cuiqingcai.com/5678.html">13.10-Scrapy通用爬虫</a></li>
                    <li><a href="https://cuiqingcai.com/5678.html">13.11-Scrapyrt的使用</a></li>
                    <li><a href="https://cuiqingcai.com/5678.html">13.12-Scrapy对接Docker</a></li>
                    <li><a href="https://cuiqingcai.com/5678.html">13.13-Scrapy爬取新浪微博</a></li>
                    <li><a href="https://cuiqingcai.com/5678.html">14-分布式爬虫</a></li>
                    <li><a href="https://cuiqingcai.com/5678.html">14.1-分布式爬虫原理</a></li>
                    <li><a href="https://cuiqingcai.com/5678.html">14.2-Scrapy-Redis源码解析</a></li>
                    <li><a href="https://cuiqingcai.com/5678.html">14.3-Scrapy分布式实现</a></li>
                    <li><a href="https://cuiqingcai.com/5678.html">14.4-Bloom Filter的对接</a></li>
                    <li><a href="https://cuiqingcai.com/5678.html">15-分布式爬虫的部署</a></li>
                    <li><a href="https://cuiqingcai.com/5678.html">15.1-Scrapyd分布式部署</a></li>
                    <li><a href="https://cuiqingcai.com/5678.html">15.2-Scrapyd-Client的使用</a></li>
                    <li><a href="https://cuiqingcai.com/5678.html">15.3-Scrapyd对接Docker</a></li>
                    <li><a href="https://cuiqingcai.com/5678.html">15.4-Scrapyd批量部署</a></li>
                    <li><a href="https://cuiqingcai.com/5678.html">15.5-Gerapy分布式管理</a></li>
                  </ul>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/01/网络爬虫宣传图.jpg" alt=""> <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/01/Python-3网格爬虫开发实战-立体图-857x1100.jpg" alt=""></p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-01-25 08:04:33" itemprop="dateCreated datePublished" datetime="2018-01-25T08:04:33+08:00">2018-01-25</time>
                </span>
                <span id="/5052.html" class="post-meta-item leancloud_visitors" data-flag-title="Python3网络爬虫开发实战教程" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>2.1k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>2 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/5006.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/5006.html" class="post-title-link" itemprop="url">在阿里云上测试Gerapy教程</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <h2 id="前几天，大才发了一个自己写的框架，介绍地址在这里，-GIT地址在这里。"><a href="#前几天，大才发了一个自己写的框架，介绍地址在这里，-GIT地址在这里。" class="headerlink" title="前几天，大才发了一个自己写的框架，介绍地址在这里， GIT地址在这里。"></a>前几天，大才发了一个自己写的框架，介绍地址在<a href="https://cuiqingcai.com/4959.html">这里</a>， GIT地址在<a href="https://github.com/Gerapy/Gerapy" target="_blank" rel="noopener">这里</a>。</h2>
                  <p>今天在阿里云上试用了一下，在这里做一个简单的说明。</p>
                  <h1 id="1、配置环境"><a href="#1、配置环境" class="headerlink" title="1、配置环境"></a>1、配置环境</h1>
                  <p>阿里云的版本是2.7.5，所以用pyenv新安装了一个3.6.4的环境，安装后使用pyenv global 3.6.4即可使用3.6.4的环境，我个人比较喜欢这样，切换自如，互不影响。 如下图： <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/01/Selection_020.png" alt=""> 接下来按照大才的文章，pip install gerapy即可，这一步没有遇到什么问题。有问题的同学可以向大才提issue。</p>
                  <h1 id="2-开启服务"><a href="#2-开启服务" class="headerlink" title="2. 开启服务"></a>2. 开启服务</h1>
                  <p>首先去阿里云的后台设置安全组 ，我的是这样： <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/01/Selection_021.png" alt=""> 然后到命令窗口对8000和6800端口放行即可。 接着执行</p>
                  <blockquote>
                    <pre><code>gerapy init cd gerapy gerapy migrate **    # 注意下一步 **  **gerapy runserver  0.0.0.0:8000 【如果你是在本地，执行 gerapy runserver即可，如果你是在阿里云上，你就要改成前面这样来执行】**
</code></pre>
                  </blockquote>
                  <p>现在在浏览器里访问：ip:8000应该就可以看到主界面了 <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/01/Selection_022.png" alt=""> 里面的各个的含义见大才的文章。</p>
                  <h1 id="3-创建项目"><a href="#3-创建项目" class="headerlink" title="3.创建项目"></a>3.创建项目</h1>
                  <p>在gerapy下的projects里面新建一个scrapy爬虫，在这里我搞的是最简单的：</p>
                  <blockquote>
                    <p>scrapy startproject gerapy_test cd gerapy_test scrapy genspider baidu www.baidu.com</p>
                  </blockquote>
                  <p>这样就是一个最简单的爬虫了，修改一个settings.py中的ROBOTSTXT_OBEY=False, 然后修改一个spiders下面的baidu.py, 这里随意，我这里设置的是输出返回的 response.url</p>
                  <h1 id="4-安装scrapyd"><a href="#4-安装scrapyd" class="headerlink" title="4.安装scrapyd"></a>4.安装scrapyd</h1>
                  <blockquote>
                    <p> pip install scrapyd</p>
                  </blockquote>
                  <p>安装好以后，命令行执行</p>
                  <blockquote>
                    <p>scrapyd</p>
                  </blockquote>
                  <p>然后浏览器中打开 ip:6800，如果你没有修改配置，应该这里会打不开，clients那里配置的时候，也应该会显示为error，就像这样： <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/01/Selection_024.png" alt=""> 后来找了一下原因发现scrapyd默认打开的也是127.0.0.1 <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/01/Selection_025.png" alt=""> 所以这个时候就要改一下配置，具体可以参考<a href="https://piaosanlang.gitbooks.io/spiders/05day/section5.5.html" target="_blank" rel="noopener">这里</a>， 我是这么修改：</p>
                  <blockquote>
                    <p>vim ~/.scrapyd.conf [scrapyd] bind_address = 0.0.0.0</p>
                  </blockquote>
                  <p>在刷新一下，就会看到前面error变成了normal <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/01/Selection_026.png" alt=""> </p>
                  <h1 id="5-打包，部署，调度"><a href="#5-打包，部署，调度" class="headerlink" title="5. 打包，部署，调度"></a>5. 打包，部署，调度</h1>
                  <p>这几步大才的文章里都有详细说明，打包完，部署，在进入clients的调度界面，点击run按钮即可跑爬虫了 <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/01/Selection_027.png" alt=""><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/01/Selection_028.png" alt=""> <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/01/Selection_029-1.png" alt=""> 可以看到输出的结果了。</p>
                  <h1 id="6-结语"><a href="#6-结语" class="headerlink" title="6.结语"></a>6.结语</h1>
                  <p>建议大家可以试着用一下，很方便，我这里只是很简单的使用了一下。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/四毛" class="author" itemprop="url" rel="index">四毛</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-01-22 18:37:28" itemprop="dateCreated datePublished" datetime="2018-01-22T18:37:28+08:00">2018-01-22</time>
                </span>
                <span id="/5006.html" class="post-meta-item leancloud_visitors" data-flag-title="在阿里云上测试Gerapy教程" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>1k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>1 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/5001.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/5001.html" class="post-title-link" itemprop="url">TensorFlow Bi-LSTM实现序列标注</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>本节我们来尝试使用 TensorFlow 搭建一个双向 LSTM (Bi-LSTM) 深度学习模型来处理序列标注问题，主要目的是学习 Bi-LSTM 的用法。</p>
                  <h2 id="Bi-LSTM"><a href="#Bi-LSTM" class="headerlink" title="Bi-LSTM"></a>Bi-LSTM</h2>
                  <p>我们知道 RNN 是可以学习到文本上下文之间的联系的，输入是上文，输出是下文，但这样的结果是模型可以根据上文推出下文，而如果输入下文，想要推出上文就没有那么简单了，为了弥补这个缺陷，我们可以让模型从两个方向来学习，这就构成了双向 RNN。在某些任务中，双向 RNN 的表现比单向 RNN 要好，本文要实现的文本分词就是其中之一。不过本文使用的模型不是简单的双向 RNN，而是 RNN 的变种 — LSTM。 <img src="https://germey.gitbooks.io/ai/content/assets/2018-01-22-00-04-15.jpg" alt=""> 如图所示为 Bi-LSTM 的基本原理，输入层的数据会经过向前和向后两个方向推算，最后输出的隐含状态再进行 concat，再作为下一层的输入，原理其实和 LSTM 是类似的，就是多了双向计算和 concat 过程。</p>
                  <h2 id="数据处理"><a href="#数据处理" class="headerlink" title="数据处理"></a>数据处理</h2>
                  <p>本文的训练和测试数据使用的是已经做好序列标注的中文文本数据。序列标注，就是给一个汉语句子作为输入，以“BEMS”组成的序列串作为输出，然后再进行切词，进而得到输入句子的划分。其中，B 代表该字是词语中的起始字，M 代表是词语中的中间字，E 代表是词语中的结束字，S 则代表是单字成词。 这里的原始数据样例如下：</p>
                  <figure class="highlight jboss-cli">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">人<span class="string">/b</span>  们<span class="string">/e</span>  常<span class="string">/s</span>  说<span class="string">/s</span>  生<span class="string">/b</span>  活<span class="string">/e</span>  是<span class="string">/s</span>  一<span class="string">/s</span>  部<span class="string">/s</span>  教<span class="string">/b</span>  科<span class="string">/m</span>  书<span class="string">/e</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里一个字对应一个标注，我们首先需要对数据进行预处理，预处理的流程如下：</p>
                  <ul>
                    <li>将句子切分</li>
                    <li>将句子的的标点符号去掉</li>
                    <li>将每个字及对应的标注切分</li>
                    <li>去掉长度为 0 的无效句子</li>
                  </ul>
                  <p>首先我们将句子切分开来并去掉标点符号，代码实现如下：</p>
                  <figure class="highlight livecodeserver">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="comment"># Read origin data</span></span><br><span class="line"><span class="keyword">text</span> = <span class="built_in">open</span>(<span class="string">'data/data.txt'</span>, encoding=<span class="string">'utf-8'</span>).<span class="built_in">read</span>()</span><br><span class="line"><span class="comment"># Get split sentences</span></span><br><span class="line"><span class="keyword">sentences</span> = re.<span class="built_in">split</span>(<span class="string">'[，。！？、‘’“”]/[bems]'</span>, <span class="keyword">text</span>)</span><br><span class="line"><span class="comment"># Filter sentences whose length is 0</span></span><br><span class="line"><span class="keyword">sentences</span> = list(<span class="built_in">filter</span>(lambda x: x.strip(), <span class="keyword">sentences</span>))</span><br><span class="line"><span class="comment"># Strip sentences</span></span><br><span class="line"><span class="keyword">sentences</span> = list(map(lambda x: x.strip(), <span class="keyword">sentences</span>))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样我们就可以将句子切分开来并做好了清洗，接下来我们还需要把每个句子中的字及标注转为 Numpy 数组，便于下一步制作词表和数据集，代码实现如下：</p>
                  <figure class="highlight livecodeserver">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import re</span><br><span class="line"><span class="comment"># To numpy array</span></span><br><span class="line"><span class="keyword">words</span>, labels = [], []</span><br><span class="line">print(<span class="string">'Start creating words and labels...'</span>)</span><br><span class="line"><span class="keyword">for</span> <span class="keyword">sentence</span> <span class="keyword">in</span> <span class="keyword">sentences</span>:</span><br><span class="line">    groups = re.findall(<span class="string">'(.)/(.)'</span>, <span class="keyword">sentence</span>)</span><br><span class="line">    arrays = np.asarray(groups)</span><br><span class="line">    <span class="keyword">words</span>.append(arrays[:, <span class="number">0</span>])</span><br><span class="line">    labels.append(arrays[:, <span class="number">1</span>])</span><br><span class="line">print(<span class="string">'Words Length'</span>, <span class="built_in">len</span>(<span class="keyword">words</span>), <span class="string">'Labels Length'</span>, <span class="built_in">len</span>(labels))</span><br><span class="line">print(<span class="string">'Words Example'</span>, <span class="keyword">words</span>[<span class="number">0</span>])</span><br><span class="line">print(<span class="string">'Labels Example'</span>, labels[<span class="number">0</span>])</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们利用正则 re 库的 findall() 方法将字及标注分开，并分别添加到 words 和 labels 数组中，运行效果如下：</p>
                  <figure class="highlight arcade">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">Words <span class="built_in">Length</span> <span class="number">321533</span> Labels <span class="built_in">Length</span> <span class="number">321533</span></span><br><span class="line">Words Example [<span class="string">'人'</span> <span class="string">'们'</span> <span class="string">'常'</span> <span class="string">'说'</span> <span class="string">'生'</span> <span class="string">'活'</span> <span class="string">'是'</span> <span class="string">'一'</span> <span class="string">'部'</span> <span class="string">'教'</span> <span class="string">'科'</span> <span class="string">'书'</span>]</span><br><span class="line">Labels Example [<span class="string">'b'</span> <span class="string">'e'</span> <span class="string">'s'</span> <span class="string">'s'</span> <span class="string">'b'</span> <span class="string">'e'</span> <span class="string">'s'</span> <span class="string">'s'</span> <span class="string">'s'</span> <span class="string">'b'</span> <span class="string">'m'</span> <span class="string">'e'</span>]</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>接下来我们有了这些数据就要开始制作词表了，词表制作起来无非就是输入词表和输出词表的不重复的正逆对应，制作词表的目的就是将输入的文字或标注转为 index，同时还能反向根据 index 获取对应的文字或标注，所以我们这里需要制作 word2id、id2word、tag2id、id2tag 四个字典。 为了解决 OOV 问题，我们还需要将无效字符也进行标注，这里我们统一取 0。制作时我们借助于 pandas 库的 Series 进行了去重和转换，另外还限制了每一句的最大长度，这里设置为 32，如果大于32，则截断，否则进行 padding，代码如下：</p>
                  <figure class="highlight vim">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">from itertools import chain</span><br><span class="line">import pandas <span class="keyword">as</span> pd</span><br><span class="line">import numpy <span class="keyword">as</span> np</span><br><span class="line"># Merge <span class="keyword">all</span> words</span><br><span class="line">all_words = <span class="keyword">list</span>(chain(*words))</span><br><span class="line"># All words <span class="keyword">to</span> Series</span><br><span class="line">all_words_sr = pd.Series(all_words)</span><br><span class="line"># Get value <span class="built_in">count</span>, <span class="built_in">index</span> changed <span class="keyword">to</span> <span class="keyword">set</span></span><br><span class="line">all_words_counts = all_words_sr.value_counts()</span><br><span class="line"># Get words <span class="keyword">set</span></span><br><span class="line">all_words_set = all_words_counts.<span class="built_in">index</span></span><br><span class="line"># Get words ids</span><br><span class="line">all_words_ids = <span class="built_in">range</span>(<span class="number">1</span>, <span class="built_in">len</span>(all_words_set) + <span class="number">1</span>)</span><br><span class="line"></span><br><span class="line"># Dict <span class="keyword">to</span> transform</span><br><span class="line">word2id = pd.Series(all_words_ids, <span class="built_in">index</span>=all_words_set)</span><br><span class="line">id2word = pd.Series(all_words_set, <span class="built_in">index</span>=all_words_ids)</span><br><span class="line"></span><br><span class="line"># Tag <span class="keyword">set</span> <span class="built_in">and</span> ids</span><br><span class="line">tags_set = [<span class="string">'x'</span>, <span class="string">'s'</span>, <span class="string">'b'</span>, <span class="string">'m'</span>, <span class="string">'e'</span>]</span><br><span class="line">tags_ids = <span class="built_in">range</span>(<span class="built_in">len</span>(tags_set))</span><br><span class="line"></span><br><span class="line"># Dict <span class="keyword">to</span> transform</span><br><span class="line">tag2id = pd.Series(tags_ids, <span class="built_in">index</span>=tags_set)</span><br><span class="line">id2tag = pd.Series(tags_set, <span class="built_in">index</span>=tag2id)</span><br><span class="line"></span><br><span class="line">max_length = <span class="number">32</span></span><br><span class="line"></span><br><span class="line">def x_transform(words):</span><br><span class="line">    ids = <span class="keyword">list</span>(word2id[words])</span><br><span class="line">    <span class="keyword">if</span> <span class="built_in">len</span>(ids) &gt;= max_length:</span><br><span class="line">        ids = ids[:max_length]</span><br><span class="line">    ids.<span class="built_in">extend</span>([<span class="number">0</span>] * (max_length - <span class="built_in">len</span>(ids)))</span><br><span class="line">    <span class="keyword">return</span> ids</span><br><span class="line"></span><br><span class="line">def y_transform(<span class="keyword">tags</span>):</span><br><span class="line">    ids = <span class="keyword">list</span>(tag2id[<span class="keyword">tags</span>])</span><br><span class="line">    <span class="keyword">if</span> <span class="built_in">len</span>(ids) &gt;= max_length:</span><br><span class="line">        ids = ids[:max_length]</span><br><span class="line">    ids.<span class="built_in">extend</span>([<span class="number">0</span>] * (max_length - <span class="built_in">len</span>(ids)))</span><br><span class="line">    <span class="keyword">return</span> ids</span><br><span class="line"></span><br><span class="line"><span class="keyword">print</span>(<span class="string">'Starting transform...'</span>)</span><br><span class="line">data_x = <span class="keyword">list</span>(<span class="keyword">map</span>(lambda <span class="keyword">x</span>: x_transform(<span class="keyword">x</span>), words))</span><br><span class="line">data_y = <span class="keyword">list</span>(<span class="keyword">map</span>(lambda <span class="keyword">y</span>: y_transform(<span class="keyword">y</span>), labels))</span><br><span class="line">data_x = np.asarray(data_x)</span><br><span class="line">data_y = np.asarray(data_y)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样我们就完成了 word2id、id2word、tag2id、id2tag 四个字典的制作，并制作好了 Numpy 数组类型的 data_x 和 data_y，这里 data_x 和 data_y 单句示例如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">Data X Example: [<span class="number">8</span>, <span class="number">43</span>, <span class="number">320</span>, <span class="number">88</span>, <span class="number">36</span>, <span class="number">198</span>, <span class="number">7</span>, <span class="number">2</span>, <span class="number">41</span>, <span class="number">163</span>, <span class="number">124</span>, <span class="number">245</span>, <span class="number">0</span>, <span class="number">0</span>, <span class="number">0</span>, <span class="number">0</span>, <span class="number">0</span>, <span class="number">0</span>, <span class="number">0</span>, <span class="number">0</span>, <span class="number">0</span>, <span class="number">0</span>, <span class="number">0</span>, <span class="number">0</span>, <span class="number">0</span>, <span class="number">0</span>, <span class="number">0</span>, <span class="number">0</span>, <span class="number">0</span>, <span class="number">0</span>, <span class="number">0</span>, <span class="number">0</span>]</span><br><span class="line">Data Y Example: [<span class="number">2</span>, <span class="number">4</span>, <span class="number">1</span>, <span class="number">1</span>, <span class="number">2</span>, <span class="number">4</span>, <span class="number">1</span>, <span class="number">1</span>, <span class="number">1</span>, <span class="number">2</span>, <span class="number">3</span>, <span class="number">4</span>, <span class="number">0</span>, <span class="number">0</span>, <span class="number">0</span>, <span class="number">0</span>, <span class="number">0</span>, <span class="number">0</span>, <span class="number">0</span>, <span class="number">0</span>, <span class="number">0</span>, <span class="number">0</span>, <span class="number">0</span>, <span class="number">0</span>, <span class="number">0</span>, <span class="number">0</span>, <span class="number">0</span>, <span class="number">0</span>, <span class="number">0</span>, <span class="number">0</span>, <span class="number">0</span>, <span class="number">0</span>]</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到数据的 x 部分，原始文字和标注结果都转化成了词表中的 index，同时不够 32 个字符就以 0 补全。 接下来我们将其保存成 pickle 文件，以备训练和测试使用：</p>
                  <figure class="highlight lua">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="built_in">print</span>(<span class="string">'Starting pickle to file...'</span>)</span><br><span class="line">with <span class="built_in">open</span>(join(<span class="built_in">path</span>, <span class="string">'data.pkl'</span>), <span class="string">'wb'</span>) as f:</span><br><span class="line">    pickle.<span class="built_in">dump</span>(data_x, f)</span><br><span class="line">    pickle.<span class="built_in">dump</span>(data_y, f)</span><br><span class="line">    pickle.<span class="built_in">dump</span>(word2id, f)</span><br><span class="line">    pickle.<span class="built_in">dump</span>(id2word, f)</span><br><span class="line">    pickle.<span class="built_in">dump</span>(tag2id, f)</span><br><span class="line">    pickle.<span class="built_in">dump</span>(id2tag, f)</span><br><span class="line"><span class="built_in">print</span>(<span class="string">'Pickle finished'</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>好，现在数据预处理部分就完成了。</p>
                  <h2 id="构造模型"><a href="#构造模型" class="headerlink" title="构造模型"></a>构造模型</h2>
                  <p>接下来我们就需要利用 pickle 文件中的数据来构建模型了，首先进行 pickle 文件的读取，然后将数据分为训练集、开发集、测试集，详细流程不再赘述，赋值为如下变量：</p>
                  <figure class="highlight autohotkey">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"># Load data</span><br><span class="line"><span class="built_in">    data_x,</span> dat<span class="built_in">a_y</span>, word2id, id2word, tag2id, id2tag = load_data()</span><br><span class="line">    # Split data</span><br><span class="line"><span class="built_in">    train_x,</span> train_y, dev_x, dev_y, test_x, test_y = get_data(dat<span class="built_in">a_x</span>, dat<span class="built_in">a_y</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>接下来我们使用 TensorFlow 自带的 Dataset 数据结构构造输入输出，利用 Dataset 我们可以构造一个 iterator 迭代器，每调用一次 get_next() 方法，我们就可以得到一个 batch，这里 Dataset 的初始化我们使用 from_tensor_slices() 方法，然后调用其 batch() 方法来初始化每个数据集的 batch_size，接着初始化同一个 iterator，并绑定到三个数据集上声明为三个 initializer，这样每调用 initializer，就会将 iterator 切换到对应的数据集上，代码实现如下：</p>
                  <figure class="highlight reasonml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"># Train <span class="keyword">and</span> dev dataset</span><br><span class="line">train_dataset = tf.data.<span class="module-access"><span class="module"><span class="identifier">Dataset</span>.</span></span>from<span class="constructor">_tensor_slices((<span class="params">train_x</span>, <span class="params">train_y</span>)</span>)</span><br><span class="line">train_dataset = train_dataset.batch(<span class="module-access"><span class="module"><span class="identifier">FLAGS</span>.</span></span>train_batch_size)</span><br><span class="line"></span><br><span class="line">dev_dataset = tf.data.<span class="module-access"><span class="module"><span class="identifier">Dataset</span>.</span></span>from<span class="constructor">_tensor_slices((<span class="params">dev_x</span>, <span class="params">dev_y</span>)</span>)</span><br><span class="line">dev_dataset = dev_dataset.batch(<span class="module-access"><span class="module"><span class="identifier">FLAGS</span>.</span></span>dev_batch_size)</span><br><span class="line"></span><br><span class="line">test_dataset = tf.data.<span class="module-access"><span class="module"><span class="identifier">Dataset</span>.</span></span>from<span class="constructor">_tensor_slices((<span class="params">test_x</span>, <span class="params">test_y</span>)</span>)</span><br><span class="line">test_dataset = test_dataset.batch(<span class="module-access"><span class="module"><span class="identifier">FLAGS</span>.</span></span>test_batch_size)</span><br><span class="line"></span><br><span class="line"># A reinitializable iterator</span><br><span class="line">iterator = tf.data.<span class="module-access"><span class="module"><span class="identifier">Iterator</span>.</span></span>from<span class="constructor">_structure(<span class="params">train_dataset</span>.<span class="params">output_types</span>, <span class="params">train_dataset</span>.<span class="params">output_shapes</span>)</span></span><br><span class="line"></span><br><span class="line">train_initializer = iterator.make<span class="constructor">_initializer(<span class="params">train_dataset</span>)</span></span><br><span class="line">dev_initializer = iterator.make<span class="constructor">_initializer(<span class="params">dev_dataset</span>)</span></span><br><span class="line">test_initializer = iterator.make<span class="constructor">_initializer(<span class="params">test_dataset</span>)</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>有了 Dataset 的 iterator，我们只需要调用一次 get_next() 方法即可得到 x 和 y_label 了，就不需要使用 placeholder 来声明了，代码如下：</p>
                  <figure class="highlight reasonml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"># Input Layer</span><br><span class="line"><span class="keyword">with</span> tf.variable<span class="constructor">_scope('<span class="params">inputs</span>')</span>:</span><br><span class="line">    x, y_label = iterator.get<span class="constructor">_next()</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>接下来我们需要实现 embedding 层，调用 TensorFlow 的 embedding_lookup 即可实现，这里没有使用 Pre Train 的 embedding，代码实现如下：</p>
                  <figure class="highlight reasonml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"># Embedding Layer</span><br><span class="line"><span class="keyword">with</span> tf.variable<span class="constructor">_scope('<span class="params">embedding</span>')</span>:</span><br><span class="line">    embedding = tf.<span class="constructor">Variable(<span class="params">tf</span>.<span class="params">random_normal</span>([<span class="params">vocab_size</span>, FLAGS.<span class="params">embedding_size</span>])</span>, dtype=tf.float32)</span><br><span class="line">inputs = tf.nn.embedding<span class="constructor">_lookup(<span class="params">embedding</span>, <span class="params">x</span>)</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>接下来我们就需要实现双向 LSTM 了，这里我们要构造一个 2 层的 Bi-LSTM 网络，实现的时候我们首先需要声明 LSTM Cell 的列表，然后调用 stack_bidirectional_rnn() 方法即可：</p>
                  <figure class="highlight reasonml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">cell_fw = <span class="literal">[<span class="identifier">lstm_cell</span>(FLAGS.<span class="identifier">num_units</span>, <span class="identifier">keep_prob</span>) <span class="identifier">for</span> <span class="identifier">_</span> <span class="identifier">in</span> <span class="identifier">range</span>(FLAGS.<span class="identifier">num_layer</span>)]</span></span><br><span class="line">cell_bw = <span class="literal">[<span class="identifier">lstm_cell</span>(FLAGS.<span class="identifier">num_units</span>, <span class="identifier">keep_prob</span>) <span class="identifier">for</span> <span class="identifier">_</span> <span class="identifier">in</span> <span class="identifier">range</span>(FLAGS.<span class="identifier">num_layer</span>)]</span></span><br><span class="line">inputs = tf.unstack(inputs, <span class="module-access"><span class="module"><span class="identifier">FLAGS</span>.</span></span>time_step, axis=<span class="number">1</span>)</span><br><span class="line">output, _, _ = tf.contrib.rnn.stack<span class="constructor">_bidirectional_rnn(<span class="params">cell_fw</span>, <span class="params">cell_bw</span>, <span class="params">inputs</span>=<span class="params">inputs</span>, <span class="params">dtype</span>=<span class="params">tf</span>.<span class="params">float32</span>)</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这个方法内部是首先对每一层的 LSTM 进行正反向计算，然后对输出隐层进行 concat，然后输入下一层再进行计算，这里值得注意的地方是，我们不能把 LSTM Cell 提前组合成 MultiRNNCell 再调用 bidirectional_dynamic_rnn() 进行计算，这样相当于只有最后一层才进行 concat，是错误的。 现在我们得到的 output 就是 Bi-LSTM 的最后输出结果了。 接下来我们需要对输出结果进行一下 stack() 操作转化为一个 Tensor，然后将其 reshape() 一下，转化为 [-1, num_units * 2] 的 shape：</p>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">output</span> = tf.stack(output, axis=<span class="number">1</span>)</span><br><span class="line"><span class="attr">output</span> = tf.reshape(output, [-<span class="number">1</span>, FLAGS.num_units * <span class="number">2</span>])</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样我们再经过一层全连接网络将维度进行转换：</p>
                  <figure class="highlight vim">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"># Output Layer</span><br><span class="line">with <span class="keyword">tf</span>.variable_scope(<span class="string">'outputs'</span>):</span><br><span class="line">    <span class="keyword">w</span> = weight([FLAGS.num_units * <span class="number">2</span>, FLAGS.category_num])</span><br><span class="line">    <span class="keyword">b</span> = bias([FLAGS.category_num])</span><br><span class="line">    <span class="keyword">y</span> = <span class="keyword">tf</span>.matmul(output, <span class="keyword">w</span>) + <span class="keyword">b</span></span><br><span class="line">    y_predict = <span class="keyword">tf</span>.cast(<span class="keyword">tf</span>.argmax(<span class="keyword">y</span>, axis=<span class="number">1</span>), <span class="keyword">tf</span>.int32)</span><br><span class="line">    <span class="keyword">print</span>(<span class="string">'Output Y'</span>, y_predict)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样得到的最后的 y_predict 即为预测结果，shape 为 [batch_size]，即每一句都得到了一个最可能的结果标注。 接下来我们需要计算一下准确率和 Loss，准确率其实就是比较 y_predict 和 y_label 的相似度，Loss 即为二者交叉熵：</p>
                  <figure class="highlight reasonml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"># Reshape y_label</span><br><span class="line">y_label_reshape = tf.cast(tf.reshape(y_label, <span class="literal">[-<span class="number">1</span>]</span>), tf.<span class="built_in">int32</span>)</span><br><span class="line"># Prediction</span><br><span class="line">correct_prediction = tf.equal(y_predict, y_label_reshape)</span><br><span class="line">accuracy = tf.reduce<span class="constructor">_mean(<span class="params">tf</span>.<span class="params">cast</span>(<span class="params">correct_prediction</span>, <span class="params">tf</span>.<span class="params">float32</span>)</span>)</span><br><span class="line"># Loss</span><br><span class="line">cross_entropy = tf.reduce<span class="constructor">_mean(<span class="params">tf</span>.<span class="params">nn</span>.<span class="params">sparse_softmax_cross_entropy_with_logits</span>(<span class="params">labels</span>=<span class="params">y_label_reshape</span>, <span class="params">logits</span>=<span class="params">tf</span>.<span class="params">cast</span>(<span class="params">y</span>, <span class="params">tf</span>.<span class="params">float32</span>)</span>))</span><br><span class="line"># Train</span><br><span class="line">train = tf.train.<span class="constructor">AdamOptimizer(FLAGS.<span class="params">learning_rate</span>)</span>.minimize(cross_entropy, global_step=global_step)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里计算交叉熵使用的是 sparse_softmax_cross_entropy_with_logits() 方法，Optimizer 使用的是 Adam。 最后指定训练过程和测试过程即可，训练过程如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">for</span> epoch <span class="keyword">in</span> range(FLAGS.epoch_num):</span><br><span class="line">    tf.train.global_step(sess, <span class="attribute">global_step_tensor</span>=global_step)</span><br><span class="line">    # Train</span><br><span class="line">    sess.<span class="builtin-name">run</span>(train_initializer)</span><br><span class="line">    <span class="keyword">for</span> <span class="keyword">step</span> <span class="keyword">in</span> range(int(train_steps)):</span><br><span class="line">        smrs, loss, acc, gstep, _ = sess.<span class="builtin-name">run</span>([summaries, cross_entropy, accuracy, global_step, train], feed_dict=&#123;keep_prob: FLAGS.keep_prob&#125;)</span><br><span class="line">        # <span class="builtin-name">Print</span> log</span><br><span class="line">        <span class="keyword">if</span> <span class="keyword">step</span> % FLAGS.steps_per_print == 0:</span><br><span class="line">            <span class="builtin-name">print</span>(<span class="string">'Global Step'</span>, gstep, <span class="string">'Step'</span>, <span class="keyword">step</span>, <span class="string">'Train Loss'</span>, loss, <span class="string">'Accuracy'</span>, acc)</span><br><span class="line"></span><br><span class="line">    <span class="keyword">if</span> epoch % FLAGS.epochs_per_dev == 0:</span><br><span class="line">        # Dev</span><br><span class="line">        sess.<span class="builtin-name">run</span>(dev_initializer)</span><br><span class="line">        <span class="keyword">for</span> <span class="keyword">step</span> <span class="keyword">in</span> range(int(dev_steps)):</span><br><span class="line">            <span class="keyword">if</span> <span class="keyword">step</span> % FLAGS.steps_per_print == 0:</span><br><span class="line">                <span class="builtin-name">print</span>(<span class="string">'Dev Accuracy'</span>, sess.<span class="builtin-name">run</span>(accuracy, feed_dict=&#123;keep_prob: 1&#125;), <span class="string">'Step'</span>, <span class="keyword">step</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里训练时首先调用了 train_initializer，将 iterator 指向训练数据，这样每调用一次 get_next()，x 和 y_label 就会被赋值为训练数据的一个 batch，接下来打印输出了 Loss，Accuracy 等内容。另外对于开发集来说，每次进行验证的时候也需要重新调用 dev_initializer，这样 iterator 会再次指向开发集，这样每调用一次 get_next()，x 和 y_label 就会被赋值为开发集的一个 batch，然后进行验证。 对于测试来说，我们可以计算其准确率，然后将测试的结果输出出来，代码实现如下：</p>
                  <figure class="highlight sas">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">sess.r<span class="meta">un(</span>test_initializer)</span><br><span class="line">for step <span class="meta">in</span><span class="meta"> range(</span>int(test_steps)):</span><br><span class="line">    x_results, y_predict_results, acc = sess.r<span class="meta">un(</span>[<span class="meta">x</span>, y_predict, accuracy], feed_dict=&#123;keep_prob: 1&#125;)</span><br><span class="line">    p<span class="meta">rint(</span><span class="string">'Test step'</span>, step, <span class="string">'Accuracy'</span>, acc)</span><br><span class="line">    y_predict_results = np.reshape(y_predict_results, x_results.shape)</span><br><span class="line">    for i <span class="meta">in</span><span class="meta"> range(</span>l<span class="meta">en(</span>x_results)):</span><br><span class="line">        x_result, y_predict_result = <span class="meta">list</span>(filter(lambda <span class="meta">x</span>: <span class="meta">x</span>, x_results[i])), <span class="meta">list</span>(</span><br><span class="line">            filter(lambda <span class="meta">x</span>: <span class="meta">x</span>, y_predict_results[i]))</span><br><span class="line">        x_text, y_predict_text = <span class="string">''</span>.jo<span class="meta">in(</span>id2word[x_result].values), <span class="string">''</span>.jo<span class="meta">in(</span>id2tag[y_predict_result].values)</span><br><span class="line">        p<span class="meta">rint(</span>x_text, y_predict_text)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里打印输出了当前测试的准确率，然后得到了测试结果，然后再结合词表将测试的真正结果打印出来即可。</p>
                  <h2 id="运行结果"><a href="#运行结果" class="headerlink" title="运行结果"></a>运行结果</h2>
                  <p>在训练过程中，我们需要构建模型图，然后调用训练部分的代码进行训练，输出结果类似如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">Global Step <span class="number">0</span> Step <span class="number">0</span> Train Loss <span class="number">1.67181</span> Accuracy <span class="number">0.1475</span></span><br><span class="line">Global Step <span class="number">100</span> Step <span class="number">100</span> Train Loss <span class="number">0.210423</span> Accuracy <span class="number">0.928125</span></span><br><span class="line">Global Step <span class="number">200</span> Step <span class="number">200</span> Train Loss <span class="number">0.208561</span> Accuracy <span class="number">0.920625</span></span><br><span class="line">Global Step <span class="number">300</span> Step <span class="number">300</span> Train Loss <span class="number">0.185281</span> Accuracy <span class="number">0.939375</span></span><br><span class="line">Global Step <span class="number">400</span> Step <span class="number">400</span> Train Loss <span class="number">0.186069</span> Accuracy <span class="number">0.938125</span></span><br><span class="line">Global Step <span class="number">500</span> Step <span class="number">500</span> Train Loss <span class="number">0.165667</span> Accuracy <span class="number">0.94375</span></span><br><span class="line">Global Step <span class="number">600</span> Step <span class="number">600</span> Train Loss <span class="number">0.201692</span> Accuracy <span class="number">0.9275</span></span><br><span class="line">Global Step <span class="number">700</span> Step <span class="number">700</span> Train Loss <span class="number">0.13299</span> Accuracy <span class="number">0.954375</span></span><br><span class="line">...</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>随着训练的进行，准确率可以达到 96% 左右。 在测试阶段，输出了当前模型的准确率及真实测试输出结果，输出结果类似如下：</p>
                  <figure class="highlight subunit">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">Test </span>step 0 Accuracy 0.946125</span><br><span class="line">据新华社北京７月９日电连日来 sbmebebmmesbes</span><br><span class="line">董新辉为自己此生不能侍奉母亲而难过 bmesbebebebmmesbe</span><br><span class="line">...</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可见测试准确率在 95% 左右，对于测试数据，此处还输出了每句话的序列标注结果，如第一行结果中，“据”字对应的标注就是 s，代表单字成词，“新”字对应的标注是 b，代表词的起始，“华”字对应标注是 m，代表词的中间，“社”字对应的标注是 e，代表结束，这样 “据”、“新华社” 就可以被分成两个词了，可见还是有一定效果的。</p>
                  <h2 id="结语"><a href="#结语" class="headerlink" title="结语"></a>结语</h2>
                  <p>本节通过搭建一个 Bi-LSTM 网络实现了序列标注，并可实现分词，准确率可达到 95% 左右，但是最主要的还是学习 Bi-LSTM 的用法，本实例代码较多，部分代码已经省略，完整代码见：<a href="https://github.com/AIDeepLearning/BiLSTMWordBreaker" target="_blank" rel="noopener">https://github.com/AIDeepLearning/BiLSTMWordBreaker</a>。</p>
                  <h2 id="参考来源"><a href="#参考来源" class="headerlink" title="参考来源"></a>参考来源</h2>
                  <ul>
                    <li><a href="http://blog.csdn.net/jerr__y/article/details/70471066" target="_blank" rel="noopener">TensorFlow入门 双端 LSTM 实现序列标注</a></li>
                    <li><a href="http://spaces.ac.cn/archives/3924/" target="_blank" rel="noopener">基于双向LSTM的seq2seq字标注</a></li>
                    <li><a href="https://www.leiphone.com/news/201711/zV7yM5W1dFrzs8W5.html" target="_blank" rel="noopener">TensorFlow全新的数据读取方式：Dataset API入门教程</a></li>
                    <li><a href="https://www.tensorflow.org/programmers_guide/datasets" target="_blank" rel="noopener">TensorFlow Importing Data</a></li>
                  </ul>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-01-22 02:18:38" itemprop="dateCreated datePublished" datetime="2018-01-22T02:18:38+08:00">2018-01-22</time>
                </span>
                <span id="/5001.html" class="post-meta-item leancloud_visitors" data-flag-title="TensorFlow Bi-LSTM实现序列标注" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>9.3k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>8 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/4989.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/4989.html" class="post-title-link" itemprop="url">Ansible扩展</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <h1 id="Ansible简介"><a href="#Ansible简介" class="headerlink" title="Ansible简介"></a>Ansible简介</h1>
                  <p> Ansible是由Python开发的一个运维工具，因为工作需要接触到Ansible，经常会集成一些东西到Ansible，所以对Ansible的了解越来越多。 那Ansible到底是什么呢？在我的理解中，原来需要登录到服务器上，然后执行一堆命令才能完成一些操作。而Ansible就是来代替我们去执行那些命令。并且可以通过Ansible控制多台机器，在机器上进行任务的编排和执行，在Ansible中称为playbook。 那Ansible是如何做到的呢？简单点说，就是Ansible将我们要执行的命令生成一个脚本，然后通过sftp将脚本上传到要执行命令的服务器上，然后在通过ssh协议，执行这个脚本并将执行结果返回。 那Ansible具体是怎么做到的呢？下面从模块和插件来看一下Ansible是如何完成一个模块的执行 PS：下面的分析都是在对Ansible有一些具体使用经验之后，通过阅读源代码进一步得出的执行结论，所以希望在看本文时，是建立在对Ansible有一定了解的基础上，最起码对于Ansible的一些概念有了解，例如inventory，module，playbooks等 </p>
                  <h1 id="Ansible模块"><a href="#Ansible模块" class="headerlink" title="Ansible模块"></a>Ansible模块</h1>
                  <p> 模块是Ansible执行的最小单位，可以是由Python编写，也可以是Shell编写，也可以是由其他语言编写。模块中定义了具体的操作步骤以及实际使用过程中所需要的参数 执行的脚本就是根据模块生成一个可执行的脚本。 那Ansible是怎么样将这个脚本上传到服务器上，然后执行获取结果的呢？ </p>
                  <h1 id="Ansible插件"><a href="#Ansible插件" class="headerlink" title="Ansible插件"></a>Ansible插件</h1>
                  <h2 id="connection插件"><a href="#connection插件" class="headerlink" title="connection插件"></a><strong>connection插件</strong></h2>
                  <p>连接插件，根据指定的ssh参数连接指定的服务器，并切提供实际执行命令的接口</p>
                  <h2 id="shell插件"><a href="#shell插件" class="headerlink" title="shell插件"></a><strong>shell插件</strong></h2>
                  <p>命令插件，根据sh类型，来生成用于connection时要执行的命令</p>
                  <h2 id="strategy插件"><a href="#strategy插件" class="headerlink" title="strategy插件"></a><strong>strategy插件</strong></h2>
                  <p>执行策略插件，默认情况下是线性插件，就是一个任务接着一个任务的向下执行，此插件将任务丢到执行器去执行。</p>
                  <h2 id="action插件"><a href="#action插件" class="headerlink" title="action插件"></a><strong>action插件</strong></h2>
                  <p>动作插件，实质就是任务模块的所有动作，如果ansible的模块没有特别编写的action插件，默认情况下是normal或者async（这两个根据模块是否async来选择），normal和async中定义的就是模块的执行步骤。例如，本地创建临时文件，上传临时文件，执行脚本，删除脚本等等，如果想在所有的模块中增加一些特殊步骤，可以通过增加action插件的方式来扩展。 </p>
                  <h1 id="Ansible执行模块流程"><a href="#Ansible执行模块流程" class="headerlink" title="Ansible执行模块流程"></a>Ansible执行模块流程</h1>
                  <ol>
                    <li>ansible命令实质是通过ansible/cli/adhoc.py来运行，同时会收集参数信息<ol>
                        <li>设置Play信息，然后通过TaskQueueManager进行run，</li>
                        <li>TaskQueueManager需要Inventory(节点仓库)，variable_manager(收集变量),options(命令行中指定的参数),stdout_callback(回调函数)</li>
                      </ol>
                    </li>
                    <li>在task_queue_manager.py中找到run中<ol>
                        <li>初始化时会设置队列</li>
                        <li>会根据options，，variable_manager，passwords等信息设置成一个PlayContext信息(playbooks/playcontext.py)</li>
                        <li>设置插件(plugins)信息callback_loader(回调), strategy_loader(执行策略), module_loader(任务模块)</li>
                        <li>通过strategy_loader（strategy插件）的run（默认的strategy类型是linear，线性执行），去按照顺序执行所有的任务（执行一个模块，可能会执行多个任务）</li>
                        <li>在strategy_loader插件run之后，会判断action类型。如果是meta类型的话会单独执行(不是具体的ansible模块时)，而其他模块时，会加载到队列_queue_task</li>
                        <li>在队列中会调用WorkerProcess去处理，在workerproces实际的run之后，会使用TaskExecutor进行执行</li>
                        <li>在TaskExecutor中会设置connection插件，并且根据task的类型（模块。或是include等）获取action插件，就是对应的模块，如果模块有自定义的执行，则会执行自定义的action，如果没有的会使用normal或者async，这个是根据是否是任务的async属性来决定</li>
                        <li>在Action插件中定义着执行的顺序，及具体操作，例如生成临时目录，生成临时脚本，所以要在统一的模式下，集成一些额外的处理时，可以重写Action的方法</li>
                        <li>通过Connection插件来执行Action的各个操作步骤</li>
                      </ol>
                    </li>
                  </ol>
                  <h1 id="扩展Ansible实例"><a href="#扩展Ansible实例" class="headerlink" title="扩展Ansible实例"></a>扩展Ansible实例</h1>
                  <h2 id="执行节点Python环境扩展"><a href="#执行节点Python环境扩展" class="headerlink" title="执行节点Python环境扩展"></a>执行节点Python环境扩展</h2>
                  <p>实际需求中，我们扩展的一些Ansible模块需要使用三方库，但每个节点中安装这些库有些不易于管理。ansible执行模块的实质就是在节点的python环境下执行生成的脚本，所以我们采取的方案是，指定节点上的Python环境，将局域网内一个python环境作为<strong>nfs</strong>共享。通过扩展Action插件，增加节点上挂载nfs，待执行结束后再将节点上的nfs卸载。具体实施步骤如下： 扩展代码：</p>
                  <p>重写ActionBase的execute_module方法</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br><span class="line">80</span><br><span class="line">81</span><br><span class="line">82</span><br><span class="line">83</span><br><span class="line">84</span><br><span class="line">85</span><br><span class="line">86</span><br><span class="line">87</span><br><span class="line">88</span><br><span class="line">89</span><br><span class="line">90</span><br><span class="line">91</span><br><span class="line">92</span><br><span class="line">93</span><br><span class="line">94</span><br><span class="line">95</span><br><span class="line">96</span><br><span class="line">97</span><br><span class="line">98</span><br><span class="line">99</span><br><span class="line">100</span><br><span class="line">101</span><br><span class="line">102</span><br><span class="line">103</span><br><span class="line">104</span><br><span class="line">105</span><br><span class="line">106</span><br><span class="line">107</span><br><span class="line">108</span><br><span class="line">109</span><br><span class="line">110</span><br><span class="line">111</span><br><span class="line">112</span><br><span class="line">113</span><br><span class="line">114</span><br><span class="line">115</span><br><span class="line">116</span><br><span class="line">117</span><br><span class="line">118</span><br><span class="line">119</span><br><span class="line">120</span><br><span class="line">121</span><br><span class="line">122</span><br><span class="line">123</span><br><span class="line">124</span><br><span class="line">125</span><br><span class="line">126</span><br><span class="line">127</span><br><span class="line">128</span><br><span class="line">129</span><br><span class="line">130</span><br><span class="line">131</span><br><span class="line">132</span><br><span class="line">133</span><br><span class="line">134</span><br><span class="line">135</span><br><span class="line">136</span><br><span class="line">137</span><br><span class="line">138</span><br><span class="line">139</span><br><span class="line">140</span><br><span class="line">141</span><br><span class="line">142</span><br><span class="line">143</span><br><span class="line">144</span><br><span class="line">145</span><br><span class="line">146</span><br><span class="line">147</span><br><span class="line">148</span><br><span class="line">149</span><br><span class="line">150</span><br><span class="line">151</span><br><span class="line">152</span><br><span class="line">153</span><br><span class="line">154</span><br><span class="line">155</span><br><span class="line">156</span><br><span class="line">157</span><br><span class="line">158</span><br><span class="line">159</span><br><span class="line">160</span><br><span class="line">161</span><br><span class="line">162</span><br><span class="line">163</span><br><span class="line">164</span><br><span class="line">165</span><br><span class="line">166</span><br><span class="line">167</span><br><span class="line">168</span><br><span class="line">169</span><br><span class="line">170</span><br><span class="line">171</span><br><span class="line">172</span><br><span class="line">173</span><br><span class="line">174</span><br><span class="line">175</span><br><span class="line">176</span><br><span class="line">177</span><br><span class="line">178</span><br><span class="line">179</span><br><span class="line">180</span><br><span class="line">181</span><br><span class="line">182</span><br><span class="line">183</span><br><span class="line">184</span><br><span class="line">185</span><br><span class="line">186</span><br><span class="line">187</span><br><span class="line">188</span><br><span class="line">189</span><br><span class="line">190</span><br><span class="line">191</span><br><span class="line">192</span><br><span class="line">193</span><br><span class="line">194</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="comment"># execute_module</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">from</span> __future__ <span class="keyword">import</span> (absolute_import, division, print_function)</span><br><span class="line">__metaclass__ = type</span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> json</span><br><span class="line"><span class="keyword">import</span> pipes</span><br><span class="line"></span><br><span class="line"><span class="keyword">from</span> ansible.compat.six <span class="keyword">import</span> text_type, iteritems</span><br><span class="line"></span><br><span class="line"><span class="keyword">from</span> ansible <span class="keyword">import</span> constants <span class="keyword">as</span> C</span><br><span class="line"><span class="keyword">from</span> ansible.errors <span class="keyword">import</span> AnsibleError</span><br><span class="line"><span class="keyword">from</span> ansible.release <span class="keyword">import</span> __version__</span><br><span class="line"></span><br><span class="line"><span class="keyword">try</span>:</span><br><span class="line">    <span class="keyword">from</span> __main__ <span class="keyword">import</span> display</span><br><span class="line"><span class="keyword">except</span> ImportError:</span><br><span class="line">    <span class="keyword">from</span> ansible.utils.display <span class="keyword">import</span> Display</span><br><span class="line">    display = Display()</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">MagicStackBase</span><span class="params">(object)</span>:</span></span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">_mount_nfs</span><span class="params">(self, ansible_nfs_src, ansible_nfs_dest)</span>:</span></span><br><span class="line">        cmd = [<span class="string">'mount'</span>,ansible_nfs_src, ansible_nfs_dest]</span><br><span class="line">        cmd = [pipes.quote(c) <span class="keyword">for</span> c <span class="keyword">in</span> cmd]</span><br><span class="line">        cmd = <span class="string">' '</span>.join(cmd)</span><br><span class="line">        result = self._low_level_execute_command(cmd=cmd, sudoable=<span class="literal">True</span>)</span><br><span class="line">        <span class="keyword">return</span> result</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">_umount_nfs</span><span class="params">(self, ansible_nfs_dest)</span>:</span></span><br><span class="line">        cmd = [<span class="string">'umount'</span>, ansible_nfs_dest]</span><br><span class="line">        cmd = [pipes.quote(c) <span class="keyword">for</span> c <span class="keyword">in</span> cmd]</span><br><span class="line">        cmd = <span class="string">' '</span>.join(cmd)</span><br><span class="line">        result = self._low_level_execute_command(cmd=cmd, sudoable=<span class="literal">True</span>)</span><br><span class="line">        <span class="keyword">return</span> result</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">_execute_module</span><span class="params">(self, module_name=None, module_args=None, tmp=None, task_vars=None, persist_files=False, delete_remote_tmp=True)</span>:</span></span><br><span class="line">        <span class="string">'''</span></span><br><span class="line"><span class="string">        Transfer and run a module along with its arguments.</span></span><br><span class="line"><span class="string">        '''</span></span><br><span class="line"></span><br><span class="line">        <span class="comment"># display.v(task_vars)</span></span><br><span class="line"></span><br><span class="line">        <span class="keyword">if</span> task_vars <span class="keyword">is</span> <span class="literal">None</span>:</span><br><span class="line">            task_vars = dict()</span><br><span class="line"></span><br><span class="line">        <span class="comment"># if a module name was not specified for this execution, use</span></span><br><span class="line">        <span class="comment"># the action from the task</span></span><br><span class="line">        <span class="keyword">if</span> module_name <span class="keyword">is</span> <span class="literal">None</span>:</span><br><span class="line">            module_name = self._task.action</span><br><span class="line">        <span class="keyword">if</span> module_args <span class="keyword">is</span> <span class="literal">None</span>:</span><br><span class="line">            module_args = self._task.args</span><br><span class="line"></span><br><span class="line">        <span class="comment"># set check mode in the module arguments, if required</span></span><br><span class="line">        <span class="keyword">if</span> self._play_context.check_mode:</span><br><span class="line">            <span class="keyword">if</span> <span class="keyword">not</span> self._supports_check_mode:</span><br><span class="line">                <span class="keyword">raise</span> AnsibleError(<span class="string">"check mode is not supported for this operation"</span>)</span><br><span class="line">            module_args[<span class="string">'_ansible_check_mode'</span>] = <span class="literal">True</span></span><br><span class="line">        <span class="keyword">else</span>:</span><br><span class="line">            module_args[<span class="string">'_ansible_check_mode'</span>] = <span class="literal">False</span></span><br><span class="line"></span><br><span class="line">        <span class="comment"># Get the connection user for permission checks</span></span><br><span class="line">        remote_user = task_vars.get(<span class="string">'ansible_ssh_user'</span>) <span class="keyword">or</span> self._play_context.remote_user</span><br><span class="line"></span><br><span class="line">        <span class="comment"># set no log in the module arguments, if required</span></span><br><span class="line">        module_args[<span class="string">'_ansible_no_log'</span>] = self._play_context.no_log <span class="keyword">or</span> C.DEFAULT_NO_TARGET_SYSLOG</span><br><span class="line"></span><br><span class="line">        <span class="comment"># set debug in the module arguments, if required</span></span><br><span class="line">        module_args[<span class="string">'_ansible_debug'</span>] = C.DEFAULT_DEBUG</span><br><span class="line"></span><br><span class="line">        <span class="comment"># let module know we are in diff mode</span></span><br><span class="line">        module_args[<span class="string">'_ansible_diff'</span>] = self._play_context.diff</span><br><span class="line"></span><br><span class="line">        <span class="comment"># let module know our verbosity</span></span><br><span class="line">        module_args[<span class="string">'_ansible_verbosity'</span>] = display.verbosity</span><br><span class="line"></span><br><span class="line">        <span class="comment"># give the module information about the ansible version</span></span><br><span class="line">        module_args[<span class="string">'_ansible_version'</span>] = __version__</span><br><span class="line"></span><br><span class="line">        <span class="comment"># set the syslog facility to be used in the module</span></span><br><span class="line">        module_args[<span class="string">'_ansible_syslog_facility'</span>] = task_vars.get(<span class="string">'ansible_syslog_facility'</span>, C.DEFAULT_SYSLOG_FACILITY)</span><br><span class="line"></span><br><span class="line">        <span class="comment"># let module know about filesystems that selinux treats specially</span></span><br><span class="line">        module_args[<span class="string">'_ansible_selinux_special_fs'</span>] = C.DEFAULT_SELINUX_SPECIAL_FS</span><br><span class="line"></span><br><span class="line">        (module_style, shebang, module_data) = self._configure_module(module_name=module_name, module_args=module_args, task_vars=task_vars)</span><br><span class="line">        <span class="keyword">if</span> <span class="keyword">not</span> shebang:</span><br><span class="line">            <span class="keyword">raise</span> AnsibleError(<span class="string">"module (%s) is missing interpreter line"</span> % module_name)</span><br><span class="line"></span><br><span class="line">        <span class="comment"># get nfs info for mount python packages</span></span><br><span class="line">        ansible_nfs_src = task_vars.get(<span class="string">"ansible_nfs_src"</span>, <span class="literal">None</span>)</span><br><span class="line">        ansible_nfs_dest = task_vars.get(<span class="string">"ansible_nfs_dest"</span>, <span class="literal">None</span>)</span><br><span class="line"></span><br><span class="line">        <span class="comment"># a remote tmp path may be necessary and not already created</span></span><br><span class="line">        remote_module_path = <span class="literal">None</span></span><br><span class="line">        args_file_path = <span class="literal">None</span></span><br><span class="line">        <span class="keyword">if</span> <span class="keyword">not</span> tmp <span class="keyword">and</span> self._late_needs_tmp_path(tmp, module_style):</span><br><span class="line">            tmp = self._make_tmp_path(remote_user)</span><br><span class="line"></span><br><span class="line">        <span class="keyword">if</span> tmp:</span><br><span class="line">            remote_module_filename = self._connection._shell.get_remote_filename(module_name)</span><br><span class="line">            remote_module_path = self._connection._shell.join_path(tmp, remote_module_filename)</span><br><span class="line">            <span class="keyword">if</span> module_style <span class="keyword">in</span> [<span class="string">'old'</span>, <span class="string">'non_native_want_json'</span>]:</span><br><span class="line">                <span class="comment"># we'll also need a temp file to hold our module arguments</span></span><br><span class="line">                args_file_path = self._connection._shell.join_path(tmp, <span class="string">'args'</span>)</span><br><span class="line"></span><br><span class="line">        <span class="keyword">if</span> remote_module_path <span class="keyword">or</span> module_style != <span class="string">'new'</span>:</span><br><span class="line">            display.debug(<span class="string">"transferring module to remote"</span>)</span><br><span class="line">            self._transfer_data(remote_module_path, module_data)</span><br><span class="line">            <span class="keyword">if</span> module_style == <span class="string">'old'</span>:</span><br><span class="line">                <span class="comment"># we need to dump the module args to a k=v string in a file on</span></span><br><span class="line">                <span class="comment"># the remote system, which can be read and parsed by the module</span></span><br><span class="line">                args_data = <span class="string">""</span></span><br><span class="line">                <span class="keyword">for</span> k,v <span class="keyword">in</span> iteritems(module_args):</span><br><span class="line">                    args_data += <span class="string">'%s=%s '</span> % (k, pipes.quote(text_type(v)))</span><br><span class="line">                self._transfer_data(args_file_path, args_data)</span><br><span class="line">            <span class="keyword">elif</span> module_style == <span class="string">'non_native_want_json'</span>:</span><br><span class="line">                self._transfer_data(args_file_path, json.dumps(module_args))</span><br><span class="line">            display.debug(<span class="string">"done transferring module to remote"</span>)</span><br><span class="line"></span><br><span class="line">        environment_string = self._compute_environment_string()</span><br><span class="line"></span><br><span class="line">        remote_files = <span class="literal">None</span></span><br><span class="line"></span><br><span class="line">        <span class="keyword">if</span> args_file_path:</span><br><span class="line">            remote_files = tmp, remote_module_path, args_file_path</span><br><span class="line">        <span class="keyword">elif</span> remote_module_path:</span><br><span class="line">            remote_files = tmp, remote_module_path</span><br><span class="line"></span><br><span class="line">        <span class="comment"># Fix permissions of the tmp path and tmp files.  This should be</span></span><br><span class="line">        <span class="comment"># called after all files have been transferred.</span></span><br><span class="line">        <span class="keyword">if</span> remote_files:</span><br><span class="line">            self._fixup_perms2(remote_files, remote_user)</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">        <span class="comment"># mount nfs</span></span><br><span class="line">        <span class="keyword">if</span> ansible_nfs_src <span class="keyword">and</span> ansible_nfs_dest:</span><br><span class="line">            result = self._mount_nfs(ansible_nfs_src, ansible_nfs_dest)</span><br><span class="line">            <span class="keyword">if</span> result[<span class="string">'rc'</span>] != <span class="number">0</span>:</span><br><span class="line">                <span class="keyword">raise</span> AnsibleError(<span class="string">"mount nfs failed!!! &#123;0&#125;"</span>.format(result[<span class="string">'stderr'</span>]))</span><br><span class="line"></span><br><span class="line">        cmd = <span class="string">""</span></span><br><span class="line">        in_data = <span class="literal">None</span></span><br><span class="line"></span><br><span class="line">        <span class="keyword">if</span> self._connection.has_pipelining <span class="keyword">and</span> self._play_context.pipelining <span class="keyword">and</span> <span class="keyword">not</span> C.DEFAULT_KEEP_REMOTE_FILES <span class="keyword">and</span> module_style == <span class="string">'new'</span>:</span><br><span class="line">            in_data = module_data</span><br><span class="line">        <span class="keyword">else</span>:</span><br><span class="line">            <span class="keyword">if</span> remote_module_path:</span><br><span class="line">                cmd = remote_module_path</span><br><span class="line"></span><br><span class="line">        rm_tmp = <span class="literal">None</span></span><br><span class="line">        <span class="keyword">if</span> tmp <span class="keyword">and</span> <span class="string">"tmp"</span> <span class="keyword">in</span> tmp <span class="keyword">and</span> <span class="keyword">not</span> C.DEFAULT_KEEP_REMOTE_FILES <span class="keyword">and</span> <span class="keyword">not</span> persist_files <span class="keyword">and</span> delete_remote_tmp:</span><br><span class="line">            <span class="keyword">if</span> <span class="keyword">not</span> self._play_context.become <span class="keyword">or</span> self._play_context.become_user == <span class="string">'root'</span>:</span><br><span class="line">                <span class="comment"># not sudoing or sudoing to root, so can cleanup files in the same step</span></span><br><span class="line">                rm_tmp = tmp</span><br><span class="line"></span><br><span class="line">        cmd = self._connection._shell.build_module_command(environment_string, shebang, cmd, arg_path=args_file_path, rm_tmp=rm_tmp)</span><br><span class="line">        cmd = cmd.strip()</span><br><span class="line">        sudoable = <span class="literal">True</span></span><br><span class="line">        <span class="keyword">if</span> module_name == <span class="string">"accelerate"</span>:</span><br><span class="line">            <span class="comment"># always run the accelerate module as the user</span></span><br><span class="line">            <span class="comment"># specified in the play, not the sudo_user</span></span><br><span class="line">            sudoable = <span class="literal">False</span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line">        res = self._low_level_execute_command(cmd, sudoable=sudoable, in_data=in_data)</span><br><span class="line"></span><br><span class="line">        <span class="comment"># umount nfs</span></span><br><span class="line">        <span class="keyword">if</span> ansible_nfs_src <span class="keyword">and</span> ansible_nfs_dest:</span><br><span class="line">            result = self._umount_nfs(ansible_nfs_dest)</span><br><span class="line">            <span class="keyword">if</span> result[<span class="string">'rc'</span>] != <span class="number">0</span>:</span><br><span class="line">                <span class="keyword">raise</span> AnsibleError(<span class="string">"umount nfs failed!!! &#123;0&#125;"</span>.format(result[<span class="string">'stderr'</span>]))</span><br><span class="line"></span><br><span class="line">        <span class="keyword">if</span> tmp <span class="keyword">and</span> <span class="string">"tmp"</span> <span class="keyword">in</span> tmp <span class="keyword">and</span> <span class="keyword">not</span> C.DEFAULT_KEEP_REMOTE_FILES <span class="keyword">and</span> <span class="keyword">not</span> persist_files <span class="keyword">and</span> delete_remote_tmp:</span><br><span class="line">            <span class="keyword">if</span> self._play_context.become <span class="keyword">and</span> self._play_context.become_user != <span class="string">'root'</span>:</span><br><span class="line">                <span class="comment"># not sudoing to root, so maybe can't delete files as that other user</span></span><br><span class="line">                <span class="comment"># have to clean up temp files as original user in a second step</span></span><br><span class="line">                tmp_rm_cmd = self._connection._shell.remove(tmp, recurse=<span class="literal">True</span>)</span><br><span class="line">                tmp_rm_res = self._low_level_execute_command(tmp_rm_cmd, sudoable=<span class="literal">False</span>)</span><br><span class="line">                tmp_rm_data = self._parse_returned_data(tmp_rm_res)</span><br><span class="line">                <span class="keyword">if</span> tmp_rm_data.get(<span class="string">'rc'</span>, <span class="number">0</span>) != <span class="number">0</span>:</span><br><span class="line">                    display.warning(<span class="string">'Error deleting remote temporary files (rc: &#123;0&#125;, stderr: &#123;1&#125;)'</span>.format(tmp_rm_res.get(<span class="string">'rc'</span>), tmp_rm_res.get(<span class="string">'stderr'</span>, <span class="string">'No error string available.'</span>)))</span><br><span class="line"></span><br><span class="line">        <span class="comment"># parse the main result</span></span><br><span class="line">        data = self._parse_returned_data(res)</span><br><span class="line"></span><br><span class="line">        <span class="comment"># pre-split stdout into lines, if stdout is in the data and there</span></span><br><span class="line">        <span class="comment"># isn't already a stdout_lines value there</span></span><br><span class="line">        <span class="keyword">if</span> <span class="string">'stdout'</span> <span class="keyword">in</span> data <span class="keyword">and</span> <span class="string">'stdout_lines'</span> <span class="keyword">not</span> <span class="keyword">in</span> data:</span><br><span class="line">            data[<span class="string">'stdout_lines'</span>] = data.get(<span class="string">'stdout'</span>, <span class="string">u''</span>).splitlines()</span><br><span class="line"></span><br><span class="line">        display.debug(<span class="string">"done with _execute_module (%s, %s)"</span> % (module_name, module_args))</span><br><span class="line">        <span class="keyword">return</span> data</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>集成到normal.py和async.py中，记住要将这两个插件在ansible.cfg中进行配置</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> __future__ import (absolute_import, division, print_function)</span><br><span class="line">__metaclass__ = type</span><br><span class="line"></span><br><span class="line"><span class="keyword">from</span> ansible.plugins.action import ActionBase</span><br><span class="line"><span class="keyword">from</span> ansible.utils.vars import merge_hash</span><br><span class="line"></span><br><span class="line"><span class="keyword">from</span> common.ansible_plugins import MagicStackBase</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">class ActionModule(MagicStackBase, ActionBase):</span><br><span class="line"></span><br><span class="line">    def <span class="builtin-name">run</span>(self, <span class="attribute">tmp</span>=None, <span class="attribute">task_vars</span>=None):</span><br><span class="line">        <span class="keyword">if</span> task_vars is None:</span><br><span class="line">            task_vars = dict()</span><br><span class="line"></span><br><span class="line">        results = super(ActionModule, self).<span class="builtin-name">run</span>(tmp, task_vars)</span><br><span class="line">        # <span class="builtin-name">remove</span> as modules might hide due <span class="keyword">to</span> nolog</span><br><span class="line">        del results[<span class="string">'invocation'</span>][<span class="string">'module_args'</span>]</span><br><span class="line">        results = merge_hash(results, self._execute_module(<span class="attribute">tmp</span>=tmp, <span class="attribute">task_vars</span>=task_vars))</span><br><span class="line">        # <span class="builtin-name">Remove</span> special fields <span class="keyword">from</span> the result, which can only be set</span><br><span class="line">        # internally by the executor engine. We <span class="keyword">do</span> this only here <span class="keyword">in</span></span><br><span class="line">        # the <span class="string">'normal'</span> action, as other action plugins may <span class="builtin-name">set</span> this.</span><br><span class="line">        #</span><br><span class="line">        # We don<span class="string">'t want modules to determine that running the module fires</span></span><br><span class="line"><span class="string">        # notify handlers.  That'</span>s <span class="keyword">for</span> the playbook <span class="keyword">to</span> decide.</span><br><span class="line">        <span class="keyword">for</span> field <span class="keyword">in</span> (<span class="string">'_ansible_notify'</span>,):</span><br><span class="line">            <span class="keyword">if</span> field <span class="keyword">in</span> results:</span><br><span class="line">                results.pop(field)</span><br><span class="line"></span><br><span class="line">        return results</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <ul>
                    <li>配置ansible.cfg，将扩展的插件指定为ansible需要的action插件</li>
                    <li>重写插件方法，重点是execute_module</li>
                    <li>执行命令中需要指定Python环境，将需要的参数添加进去nfs挂载和卸载的参数</li>
                  </ul>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">ansible <span class="number">51</span> -m mysql_db -<span class="selector-tag">a</span> <span class="string">"state=dump name=all target=/tmp/test.sql"</span> -<span class="selector-tag">i</span> hosts -u root -v -e <span class="string">"ansible_nfs_src=172.16.30.170:/web/proxy_env/lib64/python2.7/site-packages ansible_nfs_dest=/root/.pyenv/versions/2.7.10/lib/python2.7/site-packages ansible_python_interpreter=/root/.pyenv/versions/2.7.10/bin/python"</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/孟先森" class="author" itemprop="url" rel="index">孟先森</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-01-21 15:18:39" itemprop="dateCreated datePublished" datetime="2018-01-21T15:18:39+08:00">2018-01-21</time>
                </span>
                <span id="/4989.html" class="post-meta-item leancloud_visitors" data-flag-title="Ansible扩展" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>10k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>9 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/4959.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/4959.html" class="post-title-link" itemprop="url">跟繁琐的命令行说拜拜！Gerapy分布式爬虫管理框架来袭！</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <h2 id="背景"><a href="#背景" class="headerlink" title="背景"></a>背景</h2>
                  <p>用 Python 做过爬虫的小伙伴可能接触过 Scrapy，GitHub：<a href="https://github.com/scrapy/scrapy" target="_blank" rel="noopener">https://github.com/scrapy/scrapy</a>。Scrapy 的确是一个非常强大的爬虫框架，爬取效率高，扩展性好，基本上是使用 Python 开发爬虫的必备利器。如果使用 Scrapy 做爬虫，那么在爬取时，我们当然完全可以使用自己的主机来完成爬取，但当爬取量非常大的时候，我们肯定不能在自己的机器上来运行爬虫了，一个好的方法就是将 Scrapy 部署到远程服务器上来执行。 所以，这时候就出现了另一个库 Scrapyd，GitHub：<a href="https://github.com/scrapy/scrapyd" target="_blank" rel="noopener">https://github.com/scrapy/scrapyd</a>，有了它我们只需要在远程服务器上安装一个 Scrapyd，启动这个服务，就可以将我们写的 Scrapy 项目部署到远程主机上了，Scrapyd 还提供了各种操作 API，可以自由地控制 Scrapy 项目的运行，API 文档：<a href="http://scrapyd.readthedocs.io/en/stable/api.html" target="_blank" rel="noopener">http://scrapyd.readthedocs.io/en/stable/api.html</a>，例如我们将 Scrapyd 安装在 IP 为 88.88.88.88 的服务器上，然后将 Scrapy 项目部署上去，这时候我们通过请求 API 就可以来控制 Scrapy 项目的运行了，命令如下：</p>
                  <figure class="highlight groovy">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">curl <span class="string">http:</span><span class="comment">//88.88.88.88:6800/schedule.json -d project=myproject -d spider=somespider</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样就相当于启动了 myproject 项目的 somespider 爬虫，而不用我们再用命令行方式去启动爬虫，同时 Scrapyd 还提供了查看爬虫状态、取消爬虫任务、添加爬虫版本、删除爬虫版本等等的一系列 API，所以说，有了 Scrapyd，我们可以通过 API 来控制爬虫的运行，摆脱了命令行的依赖。 另外爬虫部署还是个麻烦事，因为我们需要将爬虫代码上传到远程服务器上，这个过程涉及到打包和上传两个过程，在 Scrapyd 中其实提供了这个部署的 API，叫做 addversion，但是它接受的内容是 egg 包文件，所以说要用这个接口，我们必须要把我们的 Scrapy 项目打包成 egg 文件，然后再利用文件上传的方式请求这个 addversion 接口才可以完成上传，这个过程又比较繁琐了，所以又出现了一个工具叫做 Scrapyd-Client，GitHub：<a href="https://github.com/scrapy/scrapyd-client" target="_blank" rel="noopener">https://github.com/scrapy/scrapyd-client</a>，利用它的 scrapyd-deploy 命令我们便可以完成打包和上传的两个功能，可谓是又方便了一步。 这样我们就已经解决了部署的问题，回过头来，如果我们要想实时查看服务器上 Scrapy 的运行状态，那该怎么办呢？像刚才说的，当然是请求 Scrapyd 的 API 了，如果我们想用 Python 程序来控制一下呢？我们还要用 requests 库一次次地请求这些 API ？这就太麻烦了吧，所以为了解决这个需求，Scrapyd-API 又出现了，GitHub：<a href="https://github.com/djm/python-scrapyd-api" target="_blank" rel="noopener">https://github.com/djm/python-scrapyd-api</a>，有了它我们可以只用简单的 Python 代码就可以实现 Scrapy 项目的监控和运行：</p>
                  <figure class="highlight reasonml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">from scrapyd_api import ScrapydAPI</span><br><span class="line">scrapyd = <span class="constructor">ScrapydAPI('<span class="params">http</span>:<span class="operator">/</span><span class="operator">/</span>88.888.88.88:6800')</span></span><br><span class="line">scrapyd.<span class="built_in">list</span><span class="constructor">_jobs('<span class="params">project_name</span>')</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样它的返回结果就是各个 Scrapy 项目的运行情况。 例如：</p>
                  <figure class="highlight awk">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&#123;</span><br><span class="line">    <span class="string">'pending'</span>: [</span><br><span class="line">    ],</span><br><span class="line">    <span class="string">'running'</span>: [</span><br><span class="line">        &#123;</span><br><span class="line">            <span class="string">'id'</span>: <span class="string">u'14a65...b27ce'</span>,</span><br><span class="line">            <span class="string">'spider'</span>: <span class="string">u'spider_name'</span>,</span><br><span class="line">            <span class="string">'start_time'</span>: <span class="string">u'2018-01-17 22:45:31.975358'</span></span><br><span class="line">        &#125;,</span><br><span class="line">    ],</span><br><span class="line">    <span class="string">'finished'</span>: [</span><br><span class="line">        &#123;</span><br><span class="line">            <span class="string">'id'</span>: <span class="string">'34c23...b21ba'</span>,</span><br><span class="line">            <span class="string">'spider'</span>: <span class="string">'spider_name'</span>,</span><br><span class="line">            <span class="string">'start_time'</span>: <span class="string">'2018-01-11 22:45:31.975358'</span>,</span><br><span class="line">            <span class="string">'end_time'</span>: <span class="string">'2018-01-17 14:01:18.209680'</span></span><br><span class="line">        &#125;</span><br><span class="line">    ]</span><br><span class="line">&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样我们就可以看到 Scrapy 爬虫的运行状态了。 所以，有了它们，我们可以完成的是：</p>
                  <ul>
                    <li>通过 Scrapyd 完成 Scrapy 项目的部署</li>
                    <li>通过 Scrapyd 提供的 API 来控制 Scrapy 项目的启动及状态监控</li>
                    <li>通过 Scrapyd-Client 来简化 Scrapy 项目的部署</li>
                    <li>通过 Scrapyd-API 来通过 Python 控制 Scrapy 项目</li>
                  </ul>
                  <p>是不是方便多了？ 可是？真的达到最方便了吗？肯定没有！如果这一切的一切，从 Scrapy 的部署、启动到监控、日志查看，我们只需要鼠标键盘点几下就可以完成，那岂不是美滋滋？更或者说，连 Scrapy 代码都可以帮你自动生成，那岂不是爽爆了？ 有需求就有动力，没错，Gerapy 就是为此而生的，GitHub：<a href="https://github.com/Gerapy/Gerapy" target="_blank" rel="noopener">https://github.com/Gerapy/Gerapy</a>。 本节我们就来简单了解一下 Gerapy 分布式爬虫管理框架的使用方法。</p>
                  <h2 id="安装"><a href="#安装" class="headerlink" title="安装"></a>安装</h2>
                  <p>Gerapy 是一款分布式爬虫管理框架，支持 Python 3，基于 Scrapy、Scrapyd、Scrapyd-Client、Scrapy-Redis、Scrapyd-API、Scrapy-Splash、Jinjia2、Django、Vue.js 开发，Gerapy 可以帮助我们：</p>
                  <ul>
                    <li>更方便地控制爬虫运行</li>
                    <li>更直观地查看爬虫状态</li>
                    <li>更实时地查看爬取结果</li>
                    <li>更简单地实现项目部署</li>
                    <li>更统一地实现主机管理</li>
                    <li>更轻松地编写爬虫代码</li>
                  </ul>
                  <p>安装非常简单，只需要运行 pip3 命令即可：</p>
                  <figure class="highlight cmake">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">$ pip3 <span class="keyword">install</span> gerapy</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>安装完成之后我们就可以使用 gerapy 命令了，输入 gerapy 便可以获取它的基本使用方法：</p>
                  <figure class="highlight fsharp">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">$ gerapy</span><br><span class="line">Usage:</span><br><span class="line">  gerapy init [--folder=&lt;folder&gt;]</span><br><span class="line">  gerapy migrate</span><br><span class="line">  gerapy createsuperuser</span><br><span class="line">  gerapy runserver <span class="meta">[&lt;host:port&gt;]</span></span><br><span class="line">  gerapy makemigrations</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>如果出现上述结果，就证明 Gerapy 安装成功了。</p>
                  <h2 id="初始化"><a href="#初始化" class="headerlink" title="初始化"></a>初始化</h2>
                  <p>接下来我们来开始使用 Gerapy，首先利用如下命令进行一下初始化，在任意路径下均可执行如下命令：</p>
                  <figure class="highlight elixir">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="variable">$ </span>gerapy init</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>执行完毕之后，本地便会生成一个名字为 gerapy 的文件夹，接着进入该文件夹，可以看到有一个 projects 文件夹，我们后面会用到。 紧接着执行数据库初始化命令：</p>
                  <figure class="highlight properties">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">cd</span> <span class="string">gerapy</span></span><br><span class="line"><span class="attr">gerapy</span> <span class="string">migrate</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样它就会在 gerapy 目录下生成一个 SQLite 数据库，同时建立数据库表。 接着我们只需要再运行命令启动服务就好了：</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">gerapy runserver</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样我们就可以看到 Gerapy 已经在 8000 端口上运行了。 全部的操作流程截图如下： <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/01/WX20180120-021017@2x-1100x988.png" alt=""> 接下来我们在浏览器中打开 <a href="http://localhost:8000/，就可以看到" target="_blank" rel="noopener">http://localhost:8000/，就可以看到</a> Gerapy 的主界面了： <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/01/WX20180120-021201@2x-1100x696.png" alt=""> 这里显示了主机、项目的状态，当然由于我们没有添加主机，所以所有的数目都是 0。 如果我们可以正常访问这个页面，那就证明 Gerapy 初始化都成功了。</p>
                  <h2 id="主机管理"><a href="#主机管理" class="headerlink" title="主机管理"></a>主机管理</h2>
                  <p>接下来我们可以点击左侧 Clients 选项卡，即主机管理页面，添加我们的 Scrapyd 远程服务，点击右上角的创建按钮即可添加我们需要管理的 Scrapyd 服务： <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/01/WX20180120-022115@2x-1100x657.png" alt=""> 需要添加 IP、端口，以及名称，点击创建即可完成添加，点击返回即可看到当前添加的 Scrapyd 服务列表，样例如下所示： <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/01/WX20180120-022320@2x-1100x690.png" alt=""> 这样我们可以在状态一栏看到各个 Scrapyd 服务是否可用，同时可以一目了然当前所有 Scrapyd 服务列表，另外我们还可以自由地进行编辑和删除。</p>
                  <h2 id="项目管理"><a href="#项目管理" class="headerlink" title="项目管理"></a>项目管理</h2>
                  <p>Gerapy 的核心功能当然是项目管理，在这里我们可以自由地配置、编辑、部署我们的 Scrapy 项目，点击左侧的 Projects ，即项目管理选项，我们可以看到如下空白的页面： <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/01/WX20180120-023216@2x-1100x690.png" alt=""> 假设现在我们有一个 Scrapy 项目，如果我们想要进行管理和部署，还记得初始化过程中提到的 projects 文件夹吗？这时我们只需要将项目拖动到刚才 gerapy 运行目录的 projects 文件夹下，例如我这里写好了一个 Scrapy 项目，名字叫做 zhihusite，这时把它拖动到 projects 文件夹下： <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/01/WX20180120-023937@2x.png" alt=""> 这时刷新页面，我们便可以看到 Gerapy 检测到了这个项目，同时它是不可配置、没有打包的： <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/01/WX20180120-024621@2x-1100x629.png" alt=""> 这时我们可以点击部署按钮进行打包和部署，在右下角我们可以输入打包时的描述信息，类似于 Git 的 commit 信息，然后点击打包按钮，即可发现 Gerapy 会提示打包成功，同时在左侧显示打包的结果和打包名称： <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/01/WX20180120-024945@2x-1100x283.png" alt=""> 打包成功之后，我们便可以进行部署了，我们可以选择需要部署的主机，点击后方的部署按钮进行部署，同时也可以批量选择主机进行部署，示例如下： <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/01/15-11-1100x709.jpg" alt=""> 可以发现此方法相比 Scrapyd-Client 的命令行式部署，简直不能方便更多。</p>
                  <h2 id="监控任务"><a href="#监控任务" class="headerlink" title="监控任务"></a>监控任务</h2>
                  <p>部署完毕之后就可以回到主机管理页面进行任务调度了，任选一台主机，点击调度按钮即可进入任务管理页面，此页面可以查看当前 Scrapyd 服务的所有项目、所有爬虫及运行状态： <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/01/15-12-1100x539.jpg" alt=""> 我们可以通过点击新任务、停止等按钮来实现任务的启动和停止等操作，同时也可以通过展开任务条目查看日志详情： <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/01/15-13-1100x621.jpg" alt=""> 另外我们还可以随时点击停止按钮来取消 Scrapy 任务的运行。 这样我们就可以在此页面方便地管理每个 Scrapyd 服务上的 每个 Scrapy 项目的运行了。</p>
                  <h2 id="项目编辑"><a href="#项目编辑" class="headerlink" title="项目编辑"></a>项目编辑</h2>
                  <p>同时 Gerapy 还支持项目编辑功能，有了它我们不再需要 IDE 即可完成项目的编写，我们点击项目的编辑按钮即可进入到编辑页面，如图所示： <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/01/WX20180120-030017@2x-1100x629.png" alt=""> 这样即使 Gerapy 部署在远程的服务器上，我们不方便用 IDE 打开，也不喜欢用 Vim 等编辑软件，我们可以借助于本功能方便地完成代码的编写。</p>
                  <h2 id="代码生成"><a href="#代码生成" class="headerlink" title="代码生成"></a>代码生成</h2>
                  <p>上述的项目主要针对的是我们已经写好的 Scrapy 项目，我们可以借助于 Gerapy 方便地完成编辑、部署、控制、监测等功能，而且这些项目的一些逻辑、配置都是已经写死在代码里面的，如果要修改的话，需要直接修改代码，即这些项目都是不可配置的。 在 Scrapy 中，其实提供了一个可配置化的爬虫 CrawlSpider，它可以利用一些规则来完成爬取规则和解析规则的配置，这样可配置化程度就非常高，这样我们只需要维护爬取规则、提取逻辑就可以了。如果要新增一个爬虫，我们只需要写好对应的规则即可，这类爬虫就叫做可配置化爬虫。 Gerapy 可以做到：我们写好爬虫规则，它帮我们自动生成 Scrapy 项目代码。 我们可以点击项目页面的右上角的创建按钮，增加一个可配置化爬虫，接着我们便可以在此处添加提取实体、爬取规则、抽取规则了，例如这里的解析器，我们可以配置解析成为哪个实体，每个字段使用怎样的解析方式，如 XPath 或 CSS 解析器、直接获取属性、直接添加值等多重方式，另外还可以指定处理器进行数据清洗，或直接指定正则表达式进行解析等等，通过这些流程我们可以做到任何字段的解析。 <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/01/WX20180120-031400@2x-1100x687.png" alt=""> 再比如爬取规则，我们可以指定从哪个链接开始爬取，允许爬取的域名是什么，该链接提取哪些跟进的链接，用什么解析方法来处理等等配置。通过这些配置，我们可以完成爬取规则的设置。 <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/01/WX20180120-031742@2x-1100x687.png" alt=""> 最后点击生成按钮即可完成代码的生成。 <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/01/WX20180120-032023@2x-1100x687.png" alt=""> 生成的代码示例结果如图所示，可见其结构和 Scrapy 代码是完全一致的。 <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/01/WX20180120-032249@2x-1100x570.png" alt=""> 生成代码之后，我们只需要像上述流程一样，把项目进行部署、启动就好了，不需要我们写任何一行代码，即可完成爬虫的编写、部署、控制、监测。</p>
                  <h2 id="结语"><a href="#结语" class="headerlink" title="结语"></a>结语</h2>
                  <p>以上便是 Gerapy 分布式爬虫管理框架的基本用法，如需了解更多，可以访问其 GitHub：<a href="https://github.com/Gerapy/Gerapy" target="_blank" rel="noopener">https://github.com/Gerapy/Gerapy</a>。 如果觉得此框架有不足的地方，欢迎提 Issue，也欢迎发 Pull Request 来贡献代码，如果觉得 Gerapy 有所帮助，还望赐予一个 Star！非常感谢！</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-01-20 03:34:10" itemprop="dateCreated datePublished" datetime="2018-01-20T03:34:10+08:00">2018-01-20</time>
                </span>
                <span id="/4959.html" class="post-meta-item leancloud_visitors" data-flag-title="跟繁琐的命令行说拜拜！Gerapy分布式爬虫管理框架来袭！" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>4.9k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>4 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/4934.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/4934.html" class="post-title-link" itemprop="url">TensorFlow LSTM MNIST分类</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>本节来介绍一下使用 RNN 的 LSTM 来做 MNIST 分类的方法，RNN 相比 CNN 来说，速度可能会慢，但可以节省更多的内存空间。</p>
                  <h2 id="初始化"><a href="#初始化" class="headerlink" title="初始化"></a>初始化</h2>
                  <p>首先我们可以先初始化一些变量，如学习率、节点单元数、RNN 层数等：</p>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">learning_rate</span> = <span class="number">1</span>e-<span class="number">3</span></span><br><span class="line"><span class="attr">num_units</span> = <span class="number">256</span></span><br><span class="line"><span class="attr">num_layer</span> = <span class="number">3</span></span><br><span class="line"><span class="attr">input_size</span> = <span class="number">28</span></span><br><span class="line"><span class="attr">time_step</span> = <span class="number">28</span></span><br><span class="line"><span class="attr">total_steps</span> = <span class="number">2000</span></span><br><span class="line"><span class="attr">category_num</span> = <span class="number">10</span></span><br><span class="line"><span class="attr">steps_per_validate</span> = <span class="number">100</span></span><br><span class="line"><span class="attr">steps_per_test</span> = <span class="number">500</span></span><br><span class="line"><span class="attr">batch_size</span> = tf.placeholder(tf.int32, [])</span><br><span class="line"><span class="attr">keep_prob</span> = tf.placeholder(tf.float32, [])</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>然后还需要声明一下 MNIST 数据生成器：</p>
                  <figure class="highlight clean">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">import</span> tensorflow <span class="keyword">as</span> tf</span><br><span class="line"><span class="keyword">from</span> tensorflow.examples.tutorials.mnist <span class="keyword">import</span> input_data</span><br><span class="line">mnist = input_data.read_data_sets(<span class="string">'MNIST_data/'</span>, one_hot=<span class="literal">True</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>接下来常规声明一下输入的数据，输入数据用 x 表示，标注数据用 y_label 表示：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">x = tf.placeholder(tf.<span class="built_in">float</span>32, [None, <span class="number">784</span>])</span><br><span class="line">y_label = tf.placeholder(tf.<span class="built_in">float</span>32, [None, <span class="number">10</span>])</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里输入的 x 维度是 [None, 784]，代表 batch_size 不确定，输入维度 784，y_label 同理。 接下来我们需要对输入的 x 进行 reshape 操作，因为我们需要将一张图分为多个 time_step 来输入，这样才能构建一个 RNN 序列，所以这里直接将 time_step 设成 28，这样一来 input_size 就变为了 28，batch_size 不变，所以reshape 的结果是一个三维的矩阵：</p>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">x_shape</span> = tf.reshape(x, [-<span class="number">1</span>, time_step, input_size])</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h2 id="RNN-层"><a href="#RNN-层" class="headerlink" title="RNN 层"></a>RNN 层</h2>
                  <p>接下来我们需要构建一个 RNN 模型了，这里我们使用的 RNN Cell 是 LSTMCell，而且要搭建一个三层的 RNN，所以这里还需要用到 MultiRNNCell，它的输入参数是 LSTMCell 的列表。 所以我们可以先声明一个方法用于创建 LSTMCell，方法如下：</p>
                  <figure class="highlight reasonml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">def cell(num_units):</span><br><span class="line">    cell = tf.nn.rnn_cell.<span class="constructor">BasicLSTMCell(<span class="params">num_units</span>=<span class="params">num_units</span>)</span></span><br><span class="line">    return <span class="constructor">DropoutWrapper(<span class="params">cell</span>, <span class="params">output_keep_prob</span>=<span class="params">keep_prob</span>)</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里还加入了 Dropout，来减少训练过程中的过拟合。 接下来我们再利用它来构建多层的 RNN：</p>
                  <figure class="highlight reasonml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">cells = tf.nn.rnn_cell.<span class="constructor">MultiRNNCell([<span class="params">cell</span>(<span class="params">num_units</span>)</span> for _ <span class="keyword">in</span> range(num_layer)])</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>注意这里使用了 for 循环，每循环一次新生成一个 LSTMCell，而不是直接使用乘法来扩展列表，因为这样会导致 LSTMCell 是同一个对象，导致构建完 MultiRNNCell 之后出现维度不匹配的问题。 接下来我们需要声明一个初始状态：</p>
                  <figure class="highlight reasonml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">h0 = cells.zero<span class="constructor">_state(<span class="params">batch_size</span>, <span class="params">dtype</span>=<span class="params">tf</span>.<span class="params">float32</span>)</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>然后接下来调用 dynamic_rnn() 方法即可完成模型的构建了：</p>
                  <figure class="highlight reasonml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">output, hs = tf.nn.dynamic<span class="constructor">_rnn(<span class="params">cells</span>, <span class="params">inputs</span>=<span class="params">x_shape</span>, <span class="params">initial_state</span>=<span class="params">h0</span>)</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里 inputs 的输入就是 x 做了 reshape 之后的结果，初始状态通过 initial_state 传入，其返回结果有两个，一个 output 是所有 time_step 的输出结果，赋值为 output，它是三维的，第一维长度等于 batch_size，第二维长度等于 time_step，第三维长度等于 num_units。另一个 hs 是隐含状态，是元组形式，长度即 RNN 的层数 3，每一个元素都包含了 c 和 h，即 LSTM 的两个隐含状态。 这样的话 output 的最终结果可以取最后一个 time_step 的结果，所以可以使用：</p>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">output</span> = output[:, -<span class="number">1</span>, :]</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>或者直接取隐藏状态最后一层的 h 也是相同的：</p>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">h</span> = hs[-<span class="number">1</span>].h</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>在此模型中，二者是等价的。但注意如果用于文本处理，可能由于文本长度不一，而 padding，导致二者不同。</p>
                  <h2 id="输出层"><a href="#输出层" class="headerlink" title="输出层"></a>输出层</h2>
                  <p>接下来我们再做一次线性变换和 Softmax 输出结果即可：</p>
                  <figure class="highlight reasonml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"># Output Layer</span><br><span class="line">w = tf.<span class="constructor">Variable(<span class="params">tf</span>.<span class="params">truncated_normal</span>([<span class="params">num_units</span>, <span class="params">category_num</span>], <span class="params">stddev</span>=0.1)</span>, dtype=tf.float32)</span><br><span class="line">b = tf.<span class="constructor">Variable(<span class="params">tf</span>.<span class="params">constant</span>(0.1, <span class="params">shape</span>=[<span class="params">category_num</span>])</span>, dtype=tf.float32)</span><br><span class="line">y = tf.matmul(output, w) + b</span><br><span class="line"># Loss</span><br><span class="line">cross_entropy = tf.nn.softmax<span class="constructor">_cross_entropy_with_logits(<span class="params">labels</span>=<span class="params">y_label</span>, <span class="params">logits</span>=<span class="params">y</span>)</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里的 Loss 直接调用了 softmax_cross_entropy_with_logits 先计算了 Softmax，然后计算了交叉熵。</p>
                  <h2 id="训练和评估"><a href="#训练和评估" class="headerlink" title="训练和评估"></a>训练和评估</h2>
                  <p>最后再定义训练和评估的流程即可，在训练过程中每隔一定的 step 就输出 Train Accuracy 和 Test Accuracy：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="comment"># Train</span></span><br><span class="line">train = tf.train.AdamOptimizer(<span class="attribute">learning_rate</span>=learning_rate).minimize(cross_entropy)</span><br><span class="line"></span><br><span class="line"><span class="comment"># Prediction</span></span><br><span class="line">correction_prediction = tf.equal(tf.argmax(y, <span class="attribute">axis</span>=1), tf.argmax(y_label, <span class="attribute">axis</span>=1))</span><br><span class="line">accuracy = tf.reduce_mean(tf.cast(correction_prediction, tf.float32))</span><br><span class="line"></span><br><span class="line"><span class="comment"># Train</span></span><br><span class="line">with tf.Session() as sess:</span><br><span class="line">    sess.<span class="builtin-name">run</span>(tf.global_variables_initializer())</span><br><span class="line">    <span class="keyword">for</span> <span class="keyword">step</span> <span class="keyword">in</span> range(total_steps + 1):</span><br><span class="line">        batch_x, batch_y = mnist.train.next_batch(100)</span><br><span class="line">        sess.<span class="builtin-name">run</span>(train, feed_dict=&#123;x: batch_x, y_label: batch_y, keep_prob: 0.5, batch_size: batch_x.shape[0]&#125;)</span><br><span class="line">        # Train Accuracy</span><br><span class="line">        <span class="keyword">if</span> <span class="keyword">step</span> % steps_per_validate == 0:</span><br><span class="line">            <span class="builtin-name">print</span>(<span class="string">'Train'</span>, <span class="keyword">step</span>, sess.<span class="builtin-name">run</span>(accuracy, feed_dict=&#123;x: batch_x, y_label: batch_y, keep_prob: 0.5,</span><br><span class="line">                                                               batch_size: batch_x.shape[0]&#125;))</span><br><span class="line">        # Test Accuracy</span><br><span class="line">        <span class="keyword">if</span> <span class="keyword">step</span> % steps_per_test == 0:</span><br><span class="line">            test_x, test_y = mnist.test.images, mnist.test.labels</span><br><span class="line">            <span class="builtin-name">print</span>(<span class="string">'Test'</span>, <span class="keyword">step</span>,</span><br><span class="line">                  sess.<span class="builtin-name">run</span>(accuracy, feed_dict=&#123;x: test_x, y_label: test_y, keep_prob: 1, batch_size: test_x.shape[0]&#125;))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h2 id="运行"><a href="#运行" class="headerlink" title="运行"></a>运行</h2>
                  <p>直接运行之后，只训练了几轮就可以达到 98% 的准确率：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">Train <span class="number">0</span> <span class="number">0.27</span></span><br><span class="line">Test <span class="number">0</span> <span class="number">0.2223</span></span><br><span class="line">Train <span class="number">100</span> <span class="number">0.87</span></span><br><span class="line">Train <span class="number">200</span> <span class="number">0.91</span></span><br><span class="line">Train <span class="number">300</span> <span class="number">0.94</span></span><br><span class="line">Train <span class="number">400</span> <span class="number">0.94</span></span><br><span class="line">Train <span class="number">500</span> <span class="number">0.99</span></span><br><span class="line">Test <span class="number">500</span> <span class="number">0.9595</span></span><br><span class="line">Train <span class="number">600</span> <span class="number">0.95</span></span><br><span class="line">Train <span class="number">700</span> <span class="number">0.97</span></span><br><span class="line">Train <span class="number">800</span> <span class="number">0.98</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看出来 LSTM 在做 MNIST 字符分类的任务上还是比较有效的。</p>
                  <h2 id="本节代码"><a href="#本节代码" class="headerlink" title="本节代码"></a>本节代码</h2>
                  <p>本节代码地址为：<a href="https://github.com/AIDeepLearning/LSTMClassification" target="_blank" rel="noopener">https://github.com/AIDeepLearning/LSTMClassification</a>。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-01-03 19:18:03" itemprop="dateCreated datePublished" datetime="2018-01-03T19:18:03+08:00">2018-01-03</time>
                </span>
                <span id="/4934.html" class="post-meta-item leancloud_visitors" data-flag-title="TensorFlow LSTM MNIST分类" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>3.5k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>3 分钟</span>
                </span>
              </div>
            </article>
            <script>
              document.querySelectorAll('.random').forEach(item => item.src = "https://picsum.photos/id/" + Math.floor(Math.random() * Math.floor(300)) + "/200/133")

            </script>
            <nav class="pagination">
              <a class="extend prev" rel="prev" href="/page/19/"><i class="fa fa-angle-left" aria-label="上一页"></i></a><a class="page-number" href="/">1</a><span class="space">&hellip;</span><a class="page-number" href="/page/19/">19</a><span class="page-number current">20</span><a class="page-number" href="/page/21/">21</a><span class="space">&hellip;</span><a class="page-number" href="/page/31/">31</a><a class="extend next" rel="next" href="/page/21/"><i class="fa fa-angle-right" aria-label="下一页"></i></a>
            </nav>
          </div>
          <script>
            window.addEventListener('tabs:register', () =>
            {
              let
              {
                activeClass
              } = CONFIG.comments;
              if (CONFIG.comments.storage)
              {
                activeClass = localStorage.getItem('comments_active') || activeClass;
              }
              if (activeClass)
              {
                let activeTab = document.querySelector(`a[href="#comment-${activeClass}"]`);
                if (activeTab)
                {
                  activeTab.click();
                }
              }
            });
            if (CONFIG.comments.storage)
            {
              window.addEventListener('tabs:click', event =>
              {
                if (!event.target.matches('.tabs-comment .tab-content .tab-pane')) return;
                let commentClass = event.target.classList[1];
                localStorage.setItem('comments_active', commentClass);
              });
            }

          </script>
        </div>
        <div class="toggle sidebar-toggle">
          <span class="toggle-line toggle-line-first"></span>
          <span class="toggle-line toggle-line-middle"></span>
          <span class="toggle-line toggle-line-last"></span>
        </div>
        <aside class="sidebar">
          <div class="sidebar-inner">
            <ul class="sidebar-nav motion-element">
              <li class="sidebar-nav-toc"> 文章目录 </li>
              <li class="sidebar-nav-overview"> 站点概览 </li>
            </ul>
            <!--noindex-->
            <div class="post-toc-wrap sidebar-panel">
            </div>
            <!--/noindex-->
            <div class="site-overview-wrap sidebar-panel">
              <div class="site-author motion-element" itemprop="author" itemscope itemtype="http://schema.org/Person">
                <img class="site-author-image" itemprop="image" alt="崔庆才" src="/images/avatar.png">
                <p class="site-author-name" itemprop="name">崔庆才</p>
                <div class="site-description" itemprop="description">崔庆才的个人站点，记录生活的瞬间，分享学习的心得。</div>
              </div>
              <div class="site-state-wrap motion-element">
                <nav class="site-state">
                  <div class="site-state-item site-state-posts">
                    <a href="/archives/">
                      <span class="site-state-item-count">608</span>
                      <span class="site-state-item-name">日志</span>
                    </a>
                  </div>
                  <div class="site-state-item site-state-categories">
                    <a href="/categories/">
                      <span class="site-state-item-count">24</span>
                      <span class="site-state-item-name">分类</span></a>
                  </div>
                  <div class="site-state-item site-state-tags">
                    <a href="/tags/">
                      <span class="site-state-item-count">156</span>
                      <span class="site-state-item-name">标签</span></a>
                  </div>
                </nav>
              </div>
              <div class="links-of-author motion-element">
                <span class="links-of-author-item">
                  <a href="https://github.com/Germey" title="GitHub → https:&#x2F;&#x2F;github.com&#x2F;Germey" rel="noopener" target="_blank"><i class="fab fa-github fa-fw"></i>GitHub</a>
                </span>
                <span class="links-of-author-item">
                  <a href="mailto:cqc@cuiqingcai.com.com" title="邮件 → mailto:cqc@cuiqingcai.com.com" rel="noopener" target="_blank"><i class="fa fa-envelope fa-fw"></i>邮件</a>
                </span>
                <span class="links-of-author-item">
                  <a href="https://weibo.com/cuiqingcai" title="微博 → https:&#x2F;&#x2F;weibo.com&#x2F;cuiqingcai" rel="noopener" target="_blank"><i class="fab fa-weibo fa-fw"></i>微博</a>
                </span>
                <span class="links-of-author-item">
                  <a href="https://www.zhihu.com/people/Germey" title="知乎 → https:&#x2F;&#x2F;www.zhihu.com&#x2F;people&#x2F;Germey" rel="noopener" target="_blank"><i class="fa fa-magic fa-fw"></i>知乎</a>
                </span>
              </div>
            </div>
            <div style=" width: 100%;" class="sidebar-panel sidebar-panel-image sidebar-panel-active">
              <a href="https://tutorial.lengyue.video/?coupon=12ef4b1a-a3db-11ea-bb37-0242ac130002_cqx_850" target="_blank" rel="noopener">
                <img src="https://qiniu.cuiqingcai.com/bco2a.png" style=" width: 100%;">
              </a>
            </div>
            <div style=" width: 100%;" class="sidebar-panel sidebar-panel-image sidebar-panel-active">
              <a href="http://www.ipidea.net/?utm-source=cqc&utm-keyword=?cqc" target="_blank" rel="noopener">
                <img src="https://qiniu.cuiqingcai.com/0ywun.png" style=" width: 100%;">
              </a>
            </div>
            <div class="sidebar-panel sidebar-panel-tags sidebar-panel-active">
              <h4 class="name"> 标签云 </h4>
              <div class="content">
                <a href="/tags/2048/" style="font-size: 10px;">2048</a> <a href="/tags/API/" style="font-size: 10px;">API</a> <a href="/tags/Bootstrap/" style="font-size: 11.25px;">Bootstrap</a> <a href="/tags/CDN/" style="font-size: 10px;">CDN</a> <a href="/tags/CQC/" style="font-size: 10px;">CQC</a> <a href="/tags/CSS/" style="font-size: 10px;">CSS</a> <a href="/tags/CSS-%E5%8F%8D%E7%88%AC%E8%99%AB/" style="font-size: 10px;">CSS 反爬虫</a> <a href="/tags/CV/" style="font-size: 10px;">CV</a> <a href="/tags/Django/" style="font-size: 10px;">Django</a> <a href="/tags/Eclipse/" style="font-size: 11.25px;">Eclipse</a> <a href="/tags/FTP/" style="font-size: 10px;">FTP</a> <a href="/tags/Git/" style="font-size: 10px;">Git</a> <a href="/tags/GitHub/" style="font-size: 13.75px;">GitHub</a> <a href="/tags/HTML5/" style="font-size: 10px;">HTML5</a> <a href="/tags/Hexo/" style="font-size: 10px;">Hexo</a> <a href="/tags/IT/" style="font-size: 10px;">IT</a> <a href="/tags/JSP/" style="font-size: 10px;">JSP</a> <a href="/tags/JavaScript/" style="font-size: 10px;">JavaScript</a> <a href="/tags/K8s/" style="font-size: 10px;">K8s</a> <a href="/tags/LOGO/" style="font-size: 10px;">LOGO</a> <a href="/tags/Linux/" style="font-size: 10px;">Linux</a> <a href="/tags/MIUI/" style="font-size: 10px;">MIUI</a> <a href="/tags/MongoDB/" style="font-size: 10px;">MongoDB</a> <a href="/tags/Mysql/" style="font-size: 10px;">Mysql</a> <a href="/tags/NBA/" style="font-size: 10px;">NBA</a> <a href="/tags/PHP/" style="font-size: 11.25px;">PHP</a> <a href="/tags/PS/" style="font-size: 10px;">PS</a> <a href="/tags/Pathlib/" style="font-size: 10px;">Pathlib</a> <a href="/tags/PhantomJS/" style="font-size: 10px;">PhantomJS</a> <a href="/tags/Python/" style="font-size: 15px;">Python</a> <a href="/tags/Python3/" style="font-size: 12.5px;">Python3</a> <a href="/tags/Pythonic/" style="font-size: 10px;">Pythonic</a> <a href="/tags/QQ/" style="font-size: 10px;">QQ</a> <a href="/tags/Redis/" style="font-size: 10px;">Redis</a> <a href="/tags/SAE/" style="font-size: 10px;">SAE</a> <a href="/tags/SSH/" style="font-size: 10px;">SSH</a> <a href="/tags/SVG/" style="font-size: 10px;">SVG</a> <a href="/tags/Scrapy/" style="font-size: 10px;">Scrapy</a> <a href="/tags/Scrapy-redis/" style="font-size: 10px;">Scrapy-redis</a> <a href="/tags/Scrapy%E5%88%86%E5%B8%83%E5%BC%8F/" style="font-size: 10px;">Scrapy分布式</a> <a href="/tags/Selenium/" style="font-size: 10px;">Selenium</a> <a href="/tags/TKE/" style="font-size: 10px;">TKE</a> <a href="/tags/Ubuntu/" style="font-size: 11.25px;">Ubuntu</a> <a href="/tags/VS-Code/" style="font-size: 10px;">VS Code</a> <a href="/tags/Vs-Code/" style="font-size: 10px;">Vs Code</a> <a href="/tags/Vue/" style="font-size: 11.25px;">Vue</a> <a href="/tags/Webpack/" style="font-size: 10px;">Webpack</a> <a href="/tags/Windows/" style="font-size: 10px;">Windows</a> <a href="/tags/Winpcap/" style="font-size: 10px;">Winpcap</a> <a href="/tags/WordPress/" style="font-size: 13.75px;">WordPress</a> <a href="/tags/Youtube/" style="font-size: 11.25px;">Youtube</a> <a href="/tags/android/" style="font-size: 10px;">android</a> <a href="/tags/ansible/" style="font-size: 10px;">ansible</a> <a href="/tags/cocos2d-x/" style="font-size: 10px;">cocos2d-x</a> <a href="/tags/e6/" style="font-size: 10px;">e6</a> <a href="/tags/fitvids/" style="font-size: 10px;">fitvids</a> <a href="/tags/git/" style="font-size: 11.25px;">git</a> <a href="/tags/json/" style="font-size: 10px;">json</a> <a href="/tags/js%E9%80%86%E5%90%91/" style="font-size: 10px;">js逆向</a> <a href="/tags/kubernetes/" style="font-size: 10px;">kubernetes</a> <a href="/tags/log/" style="font-size: 10px;">log</a> <a href="/tags/logging/" style="font-size: 10px;">logging</a> <a href="/tags/matlab/" style="font-size: 11.25px;">matlab</a> <a href="/tags/python/" style="font-size: 20px;">python</a> <a href="/tags/pytube/" style="font-size: 11.25px;">pytube</a> <a href="/tags/pywin32/" style="font-size: 10px;">pywin32</a> <a href="/tags/style/" style="font-size: 10px;">style</a> <a href="/tags/tomcat/" style="font-size: 10px;">tomcat</a> <a href="/tags/ubuntu/" style="font-size: 10px;">ubuntu</a> <a href="/tags/uwsgi/" style="font-size: 10px;">uwsgi</a> <a href="/tags/vsftpd/" style="font-size: 10px;">vsftpd</a> <a href="/tags/wamp/" style="font-size: 10px;">wamp</a> <a href="/tags/wineQQ/" style="font-size: 10px;">wineQQ</a> <a href="/tags/%E4%B8%83%E7%89%9B/" style="font-size: 11.25px;">七牛</a> <a href="/tags/%E4%B8%8A%E6%B5%B7/" style="font-size: 10px;">上海</a> <a href="/tags/%E4%B8%AA%E4%BA%BA%E7%BD%91%E7%AB%99/" style="font-size: 10px;">个人网站</a> <a href="/tags/%E4%B8%BB%E9%A2%98/" style="font-size: 10px;">主题</a> <a href="/tags/%E4%BA%91%E4%BA%A7%E5%93%81/" style="font-size: 10px;">云产品</a> <a href="/tags/%E4%BA%91%E5%AD%98%E5%82%A8/" style="font-size: 10px;">云存储</a> <a href="/tags/%E4%BA%AC%E4%B8%9C%E4%BA%91/" style="font-size: 10px;">京东云</a> <a href="/tags/%E4%BA%BA%E5%B7%A5%E6%99%BA%E8%83%BD/" style="font-size: 12.5px;">人工智能</a> <a href="/tags/%E4%BB%A3%E7%90%86/" style="font-size: 10px;">代理</a> <a href="/tags/%E4%BB%A3%E7%A0%81/" style="font-size: 10px;">代码</a> <a href="/tags/%E4%BB%A3%E7%A0%81%E5%88%86%E4%BA%AB%E5%9B%BE/" style="font-size: 10px;">代码分享图</a> <a href="/tags/%E4%BC%98%E5%8C%96/" style="font-size: 10px;">优化</a> <a href="/tags/%E4%BD%8D%E8%BF%90%E7%AE%97/" style="font-size: 10px;">位运算</a> <a href="/tags/%E5%85%AC%E4%BC%97%E5%8F%B7/" style="font-size: 10px;">公众号</a> <a href="/tags/%E5%88%86%E4%BA%AB/" style="font-size: 10px;">分享</a> <a href="/tags/%E5%88%86%E5%B8%83%E5%BC%8F/" style="font-size: 10px;">分布式</a> <a href="/tags/%E5%88%9B%E4%B8%9A/" style="font-size: 10px;">创业</a> <a href="/tags/%E5%89%8D%E7%AB%AF/" style="font-size: 12.5px;">前端</a> <a href="/tags/%E5%8D%9A%E5%AE%A2/" style="font-size: 10px;">博客</a> <a href="/tags/%E5%8E%9F%E7%94%9FAPP/" style="font-size: 10px;">原生APP</a> <a href="/tags/%E5%8F%8D%E7%88%AC%E8%99%AB/" style="font-size: 12.5px;">反爬虫</a> <a href="/tags/%E5%91%BD%E4%BB%A4/" style="font-size: 10px;">命令</a> <a href="/tags/%E5%93%8D%E5%BA%94%E5%BC%8F%E5%B8%83%E5%B1%80/" style="font-size: 10px;">响应式布局</a> <a href="/tags/%E5%9E%83%E5%9C%BE%E9%82%AE%E4%BB%B6/" style="font-size: 10px;">垃圾邮件</a> <a href="/tags/%E5%9F%9F%E5%90%8D%E7%BB%91%E5%AE%9A/" style="font-size: 10px;">域名绑定</a> <a href="/tags/%E5%A4%8D%E7%9B%98/" style="font-size: 10px;">复盘</a> <a href="/tags/%E5%A4%A7%E4%BC%97%E7%82%B9%E8%AF%84/" style="font-size: 10px;">大众点评</a> <a href="/tags/%E5%AD%97%E4%BD%93%E5%8F%8D%E7%88%AC%E8%99%AB/" style="font-size: 10px;">字体反爬虫</a> <a href="/tags/%E5%AD%97%E7%AC%A6%E9%97%AE%E9%A2%98/" style="font-size: 10px;">字符问题</a> <a href="/tags/%E5%AD%A6%E4%B9%A0%E6%96%B9%E6%B3%95/" style="font-size: 10px;">学习方法</a> <a href="/tags/%E5%AE%89%E5%8D%93/" style="font-size: 10px;">安卓</a> <a href="/tags/%E5%AE%9E%E7%94%A8/" style="font-size: 10px;">实用</a> <a href="/tags/%E5%B0%81%E9%9D%A2/" style="font-size: 10px;">封面</a> <a href="/tags/%E5%B4%94%E5%BA%86%E6%89%8D/" style="font-size: 18.75px;">崔庆才</a> <a href="/tags/%E5%B7%A5%E5%85%B7/" style="font-size: 12.5px;">工具</a> <a href="/tags/%E5%BC%80%E5%8F%91%E5%B7%A5%E5%85%B7/" style="font-size: 10px;">开发工具</a> <a href="/tags/%E5%BE%AE%E8%BD%AF/" style="font-size: 10px;">微软</a> <a href="/tags/%E6%80%9D%E8%80%83/" style="font-size: 10px;">思考</a> <a href="/tags/%E6%89%8B%E6%9C%BA%E8%AE%BF%E9%97%AE/" style="font-size: 10px;">手机访问</a> <a href="/tags/%E6%95%99%E7%A8%8B/" style="font-size: 10px;">教程</a> <a href="/tags/%E6%95%99%E8%82%B2/" style="font-size: 10px;">教育</a> <a href="/tags/%E6%96%B0%E4%B9%A6/" style="font-size: 12.5px;">新书</a> <a href="/tags/%E6%96%B9%E6%B3%95%E8%AE%BA/" style="font-size: 10px;">方法论</a> <a href="/tags/%E6%97%85%E6%B8%B8/" style="font-size: 10px;">旅游</a> <a href="/tags/%E6%97%A5%E5%BF%97/" style="font-size: 10px;">日志</a> <a href="/tags/%E6%9A%97%E6%97%B6%E9%97%B4/" style="font-size: 10px;">暗时间</a> <a href="/tags/%E6%9D%9C%E5%85%B0%E7%89%B9/" style="font-size: 11.25px;">杜兰特</a> <a href="/tags/%E6%A1%8C%E9%9D%A2/" style="font-size: 10px;">桌面</a> <a href="/tags/%E6%AD%8C%E5%8D%95/" style="font-size: 10px;">歌单</a> <a href="/tags/%E6%B1%9F%E5%8D%97/" style="font-size: 10px;">江南</a> <a href="/tags/%E6%B8%B8%E6%88%8F/" style="font-size: 10px;">游戏</a> <a href="/tags/%E7%84%A6%E8%99%91/" style="font-size: 10px;">焦虑</a> <a href="/tags/%E7%88%AC%E8%99%AB/" style="font-size: 16.25px;">爬虫</a> <a href="/tags/%E7%88%AC%E8%99%AB%E4%B9%A6%E7%B1%8D/" style="font-size: 11.25px;">爬虫书籍</a> <a href="/tags/%E7%8E%AF%E5%A2%83%E5%8F%98%E9%87%8F/" style="font-size: 10px;">环境变量</a> <a href="/tags/%E7%94%9F%E6%B4%BB%E7%AC%94%E8%AE%B0/" style="font-size: 10px;">生活笔记</a> <a href="/tags/%E7%99%BB%E5%BD%95/" style="font-size: 10px;">登录</a> <a href="/tags/%E7%9F%A5%E4%B9%8E/" style="font-size: 10px;">知乎</a> <a href="/tags/%E7%9F%AD%E4%BF%A1/" style="font-size: 10px;">短信</a> <a href="/tags/%E7%9F%AD%E4%BF%A1%E9%AA%8C%E8%AF%81%E7%A0%81/" style="font-size: 10px;">短信验证码</a> <a href="/tags/%E7%AC%94%E8%AE%B0%E8%BD%AF%E4%BB%B6/" style="font-size: 10px;">笔记软件</a> <a href="/tags/%E7%AF%AE%E7%BD%91/" style="font-size: 10px;">篮网</a> <a href="/tags/%E7%BA%B8%E5%BC%A0/" style="font-size: 10px;">纸张</a> <a href="/tags/%E7%BB%84%E4%BB%B6/" style="font-size: 10px;">组件</a> <a href="/tags/%E7%BD%91%E7%AB%99/" style="font-size: 10px;">网站</a> <a href="/tags/%E7%BD%91%E7%BB%9C%E7%88%AC%E8%99%AB/" style="font-size: 11.25px;">网络爬虫</a> <a href="/tags/%E7%BE%8E%E5%AD%A6/" style="font-size: 10px;">美学</a> <a href="/tags/%E8%82%89%E5%A4%B9%E9%A6%8D/" style="font-size: 10px;">肉夹馍</a> <a href="/tags/%E8%85%BE%E8%AE%AF%E4%BA%91/" style="font-size: 10px;">腾讯云</a> <a href="/tags/%E8%87%AA%E5%BE%8B/" style="font-size: 10px;">自律</a> <a href="/tags/%E8%A5%BF%E5%B0%91%E7%88%B7/" style="font-size: 10px;">西少爷</a> <a href="/tags/%E8%A7%86%E9%A2%91/" style="font-size: 10px;">视频</a> <a href="/tags/%E8%B0%B7%E6%AD%8C%E9%AA%8C%E8%AF%81%E7%A0%81/" style="font-size: 10px;">谷歌验证码</a> <a href="/tags/%E8%BF%90%E8%90%A5/" style="font-size: 10px;">运营</a> <a href="/tags/%E8%BF%9C%E7%A8%8B/" style="font-size: 10px;">远程</a> <a href="/tags/%E9%80%86%E5%90%91/" style="font-size: 10px;">逆向</a> <a href="/tags/%E9%85%8D%E7%BD%AE/" style="font-size: 10px;">配置</a> <a href="/tags/%E9%87%8D%E8%A3%85/" style="font-size: 10px;">重装</a> <a href="/tags/%E9%98%BF%E6%9D%9C/" style="font-size: 10px;">阿杜</a> <a href="/tags/%E9%9D%99%E8%A7%85/" style="font-size: 17.5px;">静觅</a> <a href="/tags/%E9%A2%A0%E8%A6%86/" style="font-size: 10px;">颠覆</a> <a href="/tags/%E9%A3%9E%E4%BF%A1/" style="font-size: 10px;">飞信</a> <a href="/tags/%E9%B8%BF%E8%92%99/" style="font-size: 10px;">鸿蒙</a>
              </div>
              <script>
                const tagsColors = ['#00a67c', '#5cb85c', '#d9534f', '#567e95', '#b37333', '#f4843d', '#15a287']
                const tagsElements = document.querySelectorAll('.sidebar-panel-tags .content a')
                tagsElements.forEach((item) =>
                {
                  item.style.backgroundColor = tagsColors[Math.floor(Math.random() * tagsColors.length)]
                })

              </script>
            </div>
            <div class="sidebar-panel sidebar-panel-categories sidebar-panel-active">
              <h4 class="name"> 分类 </h4>
              <div class="content">
                <ul class="category-list">
                  <li class="category-list-item"><a class="category-list-link" href="/categories/C-C/">C/C++</a><span class="category-list-count">23</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/HTML/">HTML</a><span class="category-list-count">14</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Java/">Java</a><span class="category-list-count">5</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/JavaScript/">JavaScript</a><span class="category-list-count">26</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Linux/">Linux</a><span class="category-list-count">15</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Markdown/">Markdown</a><span class="category-list-count">1</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Net/">Net</a><span class="category-list-count">4</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Other/">Other</a><span class="category-list-count">39</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/PHP/">PHP</a><span class="category-list-count">27</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Paper/">Paper</a><span class="category-list-count">2</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Python/">Python</a><span class="category-list-count">261</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/TypeScript/">TypeScript</a><span class="category-list-count">2</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E4%B8%AA%E4%BA%BA%E5%B1%95%E7%A4%BA/">个人展示</a><span class="category-list-count">1</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E4%B8%AA%E4%BA%BA%E6%97%A5%E8%AE%B0/">个人日记</a><span class="category-list-count">9</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E4%B8%AA%E4%BA%BA%E8%AE%B0%E5%BD%95/">个人记录</a><span class="category-list-count">4</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E4%B8%AA%E4%BA%BA%E9%9A%8F%E7%AC%94/">个人随笔</a><span class="category-list-count">15</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E5%AE%89%E8%A3%85%E9%85%8D%E7%BD%AE/">安装配置</a><span class="category-list-count">59</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E6%8A%80%E6%9C%AF%E6%9D%82%E8%B0%88/">技术杂谈</a><span class="category-list-count">88</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E6%9C%AA%E5%88%86%E7%B1%BB/">未分类</a><span class="category-list-count">1</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E7%94%9F%E6%B4%BB%E7%AC%94%E8%AE%B0/">生活笔记</a><span class="category-list-count">1</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E7%A6%8F%E5%88%A9%E4%B8%93%E5%8C%BA/">福利专区</a><span class="category-list-count">6</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E8%81%8C%E4%BD%8D%E6%8E%A8%E8%8D%90/">职位推荐</a><span class="category-list-count">2</span></li>
                </ul>
              </div>
            </div>
            <div class="sidebar-panel sidebar-panel-friends sidebar-panel-active">
              <h4 class="name"> 友情链接 </h4>
              <ul class="friends">
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/j2dub.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.findhao.net/" target="_blank" rel="noopener">FindHao</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/ou6mm.jpg">
                  </span>
                  <span class="link">
                    <a href="https://diygod.me/" target="_blank" rel="noopener">DIYgod</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/6apxu.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.51dev.com/" target="_blank" rel="noopener">IT技术社区</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://www.jankl.com/img/titleshu.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.jankl.com/" target="_blank" rel="noopener">liberalist</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/bqlbs.png">
                  </span>
                  <span class="link">
                    <a href="http://www.urselect.com/" target="_blank" rel="noopener">优社电商</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/8s88c.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.yuanrenxue.com/" target="_blank" rel="noopener">猿人学</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/2wgg5.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.yunlifang.cn/" target="_blank" rel="noopener">云立方</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/shwr6.png">
                  </span>
                  <span class="link">
                    <a href="http://lanbing510.info/" target="_blank" rel="noopener">冰蓝</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/blvoh.jpg">
                  </span>
                  <span class="link">
                    <a href="https://lengyue.me/" target="_blank" rel="noopener">冷月</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="http://qianxunclub.com/favicon.png">
                  </span>
                  <span class="link">
                    <a href="http://qianxunclub.com/" target="_blank" rel="noopener">千寻啊千寻</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/0044u.jpg">
                  </span>
                  <span class="link">
                    <a href="http://kodcloud.com/" target="_blank" rel="noopener">可道云</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/ygnpn.jpg">
                  </span>
                  <span class="link">
                    <a href="http://www.kunkundashen.cn/" target="_blank" rel="noopener">坤坤大神</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/22uv1.png">
                  </span>
                  <span class="link">
                    <a href="http://www.cenchong.com/" target="_blank" rel="noopener">岑冲博客</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/ev9kl.png">
                  </span>
                  <span class="link">
                    <a href="http://www.zxiaoji.com/" target="_blank" rel="noopener">张小鸡</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://www.503error.com/favicon.ico">
                  </span>
                  <span class="link">
                    <a href="https://www.503error.com/" target="_blank" rel="noopener">张志明个人博客</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/x714o.jpg">
                  </span>
                  <span class="link">
                    <a href="http://www.hubwiz.com/" target="_blank" rel="noopener">汇智网</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/129d8.png">
                  </span>
                  <span class="link">
                    <a href="https://www.bysocket.com/" target="_blank" rel="noopener">泥瓦匠BYSocket</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://www.xiongge.club/favicon.ico">
                  </span>
                  <span class="link">
                    <a href="https://www.xiongge.club/" target="_blank" rel="noopener">熊哥club</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/3w4fe.png">
                  </span>
                  <span class="link">
                    <a href="https://zerlong.com/" target="_blank" rel="noopener">知语</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/44hxf.png">
                  </span>
                  <span class="link">
                    <a href="http://redstonewill.com/" target="_blank" rel="noopener">红色石头</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/8g1fk.jpg">
                  </span>
                  <span class="link">
                    <a href="http://www.laodong.me/" target="_blank" rel="noopener">老董博客</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/wkaus.jpg">
                  </span>
                  <span class="link">
                    <a href="https://zhaoshuai.me/" target="_blank" rel="noopener">碎念</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/pgo0r.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.chenwenguan.com/" target="_blank" rel="noopener">陈文管的博客</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/kk82a.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.lxlinux.net/" target="_blank" rel="noopener">良许Linux教程网</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/lj0t2.jpg">
                  </span>
                  <span class="link">
                    <a href="https://tanqingbo.cn/" target="_blank" rel="noopener">IT码农</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/i8cdr.png">
                  </span>
                  <span class="link">
                    <a href="https://junyiseo.com/" target="_blank" rel="noopener">均益个人博客</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/chwv2.png">
                  </span>
                  <span class="link">
                    <a href="https://brucedone.com/" target="_blank" rel="noopener">大鱼的鱼塘</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/2y43o.png">
                  </span>
                  <span class="link">
                    <a href="http://bbs.nightteam.cn/" target="_blank" rel="noopener">夜幕爬虫安全论坛</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/zvc3w.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.weishidong.com/" target="_blank" rel="noopener">韦世东的技术专栏</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/ebudy.jpg">
                  </span>
                  <span class="link">
                    <a href="https://chuanjiabing.com/" target="_blank" rel="noopener">穿甲兵技术社区</a>
                  </span>
                </li>
              </ul>
            </div>
          </div>
        </aside>
        <div id="sidebar-dimmer"></div>
      </div>
    </main>
    <footer class="footer">
      <div class="footer-inner">
        <div class="copyright"> &copy; <span itemprop="copyrightYear">2021</span>
          <span class="with-love">
            <i class="fa fa-heart"></i>
          </span>
          <span class="author" itemprop="copyrightHolder">崔庆才丨静觅</span>
          <span class="post-meta-divider">|</span>
          <span class="post-meta-item-icon">
            <i class="fa fa-chart-area"></i>
          </span>
          <span title="站点总字数">2.6m</span>
          <span class="post-meta-divider">|</span>
          <span class="post-meta-item-icon">
            <i class="fa fa-coffee"></i>
          </span>
          <span title="站点阅读时长">39:54</span>
        </div>
        <div class="powered-by">由 <a href="https://hexo.io/" class="theme-link" rel="noopener" target="_blank">Hexo</a> & <a href="https://pisces.theme-next.org/" class="theme-link" rel="noopener" target="_blank">NexT.Pisces</a> 强力驱动 </div>
        <div class="beian"><a href="https://beian.miit.gov.cn/" rel="noopener" target="_blank">京ICP备18015597号-1 </a>
        </div>
        <script>
          (function ()
          {
            function leancloudSelector(url)
            {
              url = encodeURI(url);
              return document.getElementById(url).querySelector('.leancloud-visitors-count');
            }

            function addCount(Counter)
            {
              var visitors = document.querySelector('.leancloud_visitors');
              var url = decodeURI(visitors.id);
              var title = visitors.dataset.flagTitle;
              Counter('get', '/classes/Counter?where=' + encodeURIComponent(JSON.stringify(
              {
                url
              }))).then(response => response.json()).then((
              {
                results
              }) =>
              {
                if (results.length > 0)
                {
                  var counter = results[0];
                  leancloudSelector(url).innerText = counter.time + 1;
                  Counter('put', '/classes/Counter/' + counter.objectId,
                  {
                    time:
                    {
                      '__op': 'Increment',
                      'amount': 1
                    }
                  }).catch(error =>
                  {
                    console.error('Failed to save visitor count', error);
                  });
                }
                else
                {
                  Counter('post', '/classes/Counter',
                  {
                    title,
                    url,
                    time: 1
                  }).then(response => response.json()).then(() =>
                  {
                    leancloudSelector(url).innerText = 1;
                  }).catch(error =>
                  {
                    console.error('Failed to create', error);
                  });
                }
              }).catch(error =>
              {
                console.error('LeanCloud Counter Error', error);
              });
            }

            function showTime(Counter)
            {
              var visitors = document.querySelectorAll('.leancloud_visitors');
              var entries = [...visitors].map(element =>
              {
                return decodeURI(element.id);
              });
              Counter('get', '/classes/Counter?where=' + encodeURIComponent(JSON.stringify(
              {
                url:
                {
                  '$in': entries
                }
              }))).then(response => response.json()).then((
              {
                results
              }) =>
              {
                for (let url of entries)
                {
                  let target = results.find(item => item.url === url);
                  leancloudSelector(url).innerText = target ? target.time : 0;
                }
              }).catch(error =>
              {
                console.error('LeanCloud Counter Error', error);
              });
            }
            let
            {
              app_id,
              app_key,
              server_url
            } = {
              "enable": true,
              "app_id": "6X5dRQ0pnPWJgYy8SXOg0uID-gzGzoHsz",
              "app_key": "ziLDVEy73ne5HtFTiGstzHMS",
              "server_url": "https://6x5drq0p.lc-cn-n1-shared.com",
              "security": false
            };

            function fetchData(api_server)
            {
              var Counter = (method, url, data) =>
              {
                return fetch(`${api_server}/1.1${url}`,
                {
                  method,
                  headers:
                  {
                    'X-LC-Id': app_id,
                    'X-LC-Key': app_key,
                    'Content-Type': 'application/json',
                  },
                  body: JSON.stringify(data)
                });
              };
              if (CONFIG.page.isPost)
              {
                if (CONFIG.hostname !== location.hostname) return;
                addCount(Counter);
              }
              else if (document.querySelectorAll('.post-title-link').length >= 1)
              {
                showTime(Counter);
              }
            }
            let api_server = app_id.slice(-9) !== '-MdYXbMMI' ? server_url : `https://${app_id.slice(0, 8).toLowerCase()}.api.lncldglobal.com`;
            if (api_server)
            {
              fetchData(api_server);
            }
            else
            {
              fetch('https://app-router.leancloud.cn/2/route?appId=' + app_id).then(response => response.json()).then((
              {
                api_server
              }) =>
              {
                fetchData('https://' + api_server);
              });
            }
          })();

        </script>
      </div>
      <div class="footer-stat">
        <span id="cnzz_stat_icon_1279355174"></span>
        <script type="text/javascript">
          document.write(unescape("%3Cspan id='cnzz_stat_icon_1279355174'%3E%3C/span%3E%3Cscript src='https://v1.cnzz.com/z_stat.php%3Fid%3D1279355174%26online%3D1%26show%3Dline' type='text/javascript'%3E%3C/script%3E"));

        </script>
      </div>
    </footer>
  </div>
  <script src="//cdn.jsdelivr.net/npm/animejs@3.2.1/lib/anime.min.js"></script>
  <script src="//cdn.jsdelivr.net/npm/pangu@4/dist/browser/pangu.min.js"></script>
  <script src="/js/utils.js"></script>
  <script src="/.js"></script>
  <script src="/js/schemes/pisces.js"></script>
  <script src="/.js"></script>
  <script src="/js/next-boot.js"></script>
  <script src="/.js"></script>
  <script>
    (function ()
    {
      var canonicalURL, curProtocol;
      //Get the <link> tag
      var x = document.getElementsByTagName("link");
      //Find the last canonical URL
      if (x.length > 0)
      {
        for (i = 0; i < x.length; i++)
        {
          if (x[i].rel.toLowerCase() == 'canonical' && x[i].href)
          {
            canonicalURL = x[i].href;
          }
        }
      }
      //Get protocol
      if (!canonicalURL)
      {
        curProtocol = window.location.protocol.split(':')[0];
      }
      else
      {
        curProtocol = canonicalURL.split(':')[0];
      }
      //Get current URL if the canonical URL does not exist
      if (!canonicalURL) canonicalURL = window.location.href;
      //Assign script content. Replace current URL with the canonical URL
      ! function ()
      {
        var e = /([http|https]:\/\/[a-zA-Z0-9\_\.]+\.baidu\.com)/gi,
          r = canonicalURL,
          t = document.referrer;
        if (!e.test(r))
        {
          var n = (String(curProtocol).toLowerCase() === 'https') ? "https://sp0.baidu.com/9_Q4simg2RQJ8t7jm9iCKT-xh_/s.gif" : "//api.share.baidu.com/s.gif";
          t ? (n += "?r=" + encodeURIComponent(document.referrer), r && (n += "&l=" + r)) : r && (n += "?l=" + r);
          var i = new Image;
          i.src = n
        }
      }(window);
    })();

  </script>
  <script src="/js/local-search.js"></script>
  <script src="/.js"></script>
</body>

</html>
